diff options
Diffstat (limited to 'tests')
-rw-r--r-- | tests/.gitignore | 1 | ||||
-rw-r--r-- | tests/Makefile | 28 | ||||
-rw-r--r-- | tests/frametest.c | 297 | ||||
-rw-r--r-- | tests/fullbench.c | 97 | ||||
-rw-r--r-- | tests/fuzzer.c | 321 | ||||
-rw-r--r-- | tests/test-lz4-list.py | 282 |
6 files changed, 785 insertions, 241 deletions
diff --git a/tests/.gitignore b/tests/.gitignore index c4f9092..0d13df8 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -15,6 +15,7 @@ checkFrame # test artefacts tmp* versionsTest +lz4_all.c # local tests afl diff --git a/tests/Makefile b/tests/Makefile index 67514e4..422baba 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -114,7 +114,7 @@ clean: frametest$(EXT) frametest32$(EXT) \ fasttest$(EXT) roundTripTest$(EXT) \ datagen$(EXT) checkTag$(EXT) \ - frameTest$(EXT) + frameTest$(EXT) lz4_all.c @$(RM) -rf $(TESTDIR) @echo Cleaning completed @@ -122,10 +122,13 @@ clean: versionsTest: $(PYTHON) test-lz4-versions.py +.PHONY: listTest +listTest: lz4 + QEMU_SYS=$(QEMU_SYS) $(PYTHON) test-lz4-list.py + checkTag: checkTag.c $(LZ4DIR)/lz4.h $(CC) $(FLAGS) $< -o $@$(EXT) - #----------------------------------------------------------------------------- # validated only for Linux, OSX, BSD, Hurd and Solaris targets #----------------------------------------------------------------------------- @@ -150,19 +153,19 @@ list: @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs .PHONY: test -test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-install test-amalgamation +test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-install test-amalgamation listTest .PHONY: test32 test32: CFLAGS+=-m32 test32: test -.PHONY: test-amalgamation -test-amalgamation: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c - cat $(LZ4DIR)/lz4.c > lz4_all.c - cat $(LZ4DIR)/lz4hc.c >> lz4_all.c - cat $(LZ4DIR)/lz4frame.c >> lz4_all.c - $(CC) -I$(LZ4DIR) -c lz4_all.c - $(RM) lz4_all.c +test-amalgamation: lz4_all.o + +lz4_all.o: lz4_all.c + $(CC) $(CFLAGS) $(CPPFLAGS) -c $^ -o $@ + +lz4_all.c: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c + cat $^ > $@ test-install: lz4 lib liblz4.pc lz4_root=.. ./test_install.sh @@ -373,6 +376,7 @@ test-lz4-testmode: lz4 datagen @echo "\n ---- non-existing source ----" ! $(LZ4) file-does-not-exist ! $(LZ4) -f file-does-not-exist + ! $(LZ4) -t file-does-not-exist ! $(LZ4) -fm file1-dne file2-dne @$(RM) tmp-tlt tmp-tlt1 tmp-tlt2 tmp-tlt2.lz4 @@ -444,7 +448,7 @@ test-fuzzer32: CFLAGS += -m32 test-fuzzer32: test-fuzzer test-frametest: frametest - ./frametest $(FUZZER_TIME) + ./frametest -v $(FUZZER_TIME) test-frametest32: CFLAGS += -m32 test-frametest32: test-frametest @@ -463,6 +467,8 @@ test-mem: lz4 datagen fuzzer frametest fullbench valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 ftmdg7M valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 ftmdg7M ftmdg16K2 valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq ftmdg7M $(VOID) + valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --list -m ftm*.lz4 + valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --list -m -v ftm*.lz4 $(RM) ftm* valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i64 -t1 valgrind --leak-check=yes --error-exitcode=1 ./frametest -i256 diff --git a/tests/frametest.c b/tests/frametest.c index bf95beb..1b932e4 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -41,11 +41,12 @@ #include <string.h> /* strcmp */ #include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */ #include <assert.h> -#include "lz4frame.h" /* include multiple times to test correctness/safety */ +#include "lz4frame.h" /* included multiple times to test correctness/safety */ #include "lz4frame.h" #define LZ4F_STATIC_LINKING_ONLY #include "lz4frame.h" #include "lz4frame.h" +#define LZ4_STATIC_LINKING_ONLY /* LZ4_DISTANCE_MAX */ #include "lz4.h" /* LZ4_VERSION_STRING */ #define XXH_STATIC_LINKING_ONLY #include "xxhash.h" /* XXH64 */ @@ -150,8 +151,7 @@ static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, dou size_t const length = MIN(lengthRand, bufferSize - pos); size_t const end = pos + length; while (pos < end) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5); - } - } + } } } @@ -541,7 +541,7 @@ int basicTests(U32 seed, double compressibility) cdict, NULL) ); DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n", (unsigned)dictSize, (unsigned)cSizeWithDict); - if (cSizeWithDict >= cSizeNoDict) goto _output_error; /* must be more efficient */ + if ((LZ4_DISTANCE_MAX > dictSize) && (cSizeWithDict >= cSizeNoDict)) goto _output_error; /* must be more efficient */ crcOrig = XXH64(CNBuffer, dictSize, 0); DISPLAYLEVEL(3, "LZ4F_decompress_usingDict : "); @@ -703,8 +703,8 @@ int basicTests(U32 seed, double compressibility) while (ip < iend) { unsigned nbBits = FUZ_rand(&randState) % maxBits; size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; - size_t oSize = oend-op; - if (iSize > (size_t)(iend-ip)) iSize = iend-ip; + size_t oSize = (size_t)(oend-op); + if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip); CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); op += oSize; ip += iSize; @@ -722,8 +722,8 @@ int basicTests(U32 seed, double compressibility) while (ip < iend) { unsigned const nbBits = FUZ_rand(&randState) % maxBits; size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; - size_t oSize = oend-op; - if (iSize > (size_t)(iend-ip)) iSize = iend-ip; + size_t oSize = (size_t)(oend-op); + if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip); CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); op += oSize; ip += iSize; @@ -739,7 +739,7 @@ int basicTests(U32 seed, double compressibility) while (ip < iend) { size_t iSize = 10; size_t oSize = 10; - if (iSize > (size_t)(iend-ip)) iSize = iend-ip; + if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip); CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); op += oSize; ip += iSize; @@ -763,55 +763,173 @@ _output_error: } -static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, unsigned nonContiguous) +typedef enum { o_contiguous, o_noncontiguous, o_overwrite } o_scenario_e; + +static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, o_scenario_e o_scenario) { - size_t p=0; - const BYTE* b1=(const BYTE*)buff1; - const BYTE* b2=(const BYTE*)buff2; - DISPLAY("locateBuffDiff: looking for error position \n"); - if (nonContiguous) { - DISPLAY("mode %u: non-contiguous output (%u bytes), cannot search \n", - nonContiguous, (unsigned)size); - return; + if (displayLevel >= 5) { + size_t p=0; + const BYTE* b1=(const BYTE*)buff1; + const BYTE* b2=(const BYTE*)buff2; + DISPLAY("locateBuffDiff: looking for error position \n"); + if (o_scenario != o_contiguous) { + DISPLAY("mode %i: non-contiguous output (%u bytes), cannot search \n", + (int)o_scenario, (unsigned)size); + return; + } + while (p < size && b1[p]==b2[p]) p++; + if (p != size) { + DISPLAY("Error at pos %i/%i : %02X != %02X \n", (int)p, (int)size, b1[p], b2[p]); + } } - while (p < size && b1[p]==b2[p]) p++; - if (p != size) { - DISPLAY("Error at pos %i/%i : %02X != %02X \n", (int)p, (int)size, b1[p], b2[p]); +} + +# define EXIT_MSG(...) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ + DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); exit(1); } +# undef CHECK +# define CHECK(cond, ...) { if (cond) { EXIT_MSG(__VA_ARGS__); } } + + +size_t test_lz4f_decompression_wBuffers( + const void* cSrc, size_t cSize, + void* dst, size_t dstCapacity, o_scenario_e o_scenario, + const void* srcRef, size_t decompressedSize, + U64 crcOrig, + U32* const randState, + LZ4F_dctx* const dCtx, + U32 seed, U32 testNb) +{ + const BYTE* ip = (const BYTE*)cSrc; + const BYTE* const iend = ip + cSize; + + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + dstCapacity; + + unsigned const suggestedBits = FUZ_highbit((U32)cSize); + unsigned const maxBits = MAX(3, suggestedBits); + size_t totalOut = 0; + size_t moreToFlush = 0; + XXH64_state_t xxh64; + XXH64_reset(&xxh64, 1); + assert(ip < iend); + while (ip < iend) { + unsigned const nbBitsI = (FUZ_rand(randState) % (maxBits-1)) + 1; + unsigned const nbBitsO = (FUZ_rand(randState) % (maxBits)) + 1; + size_t const iSizeCand = (FUZ_rand(randState) & ((1<<nbBitsI)-1)) + 1; + size_t const iSizeMax = MIN(iSizeCand, (size_t)(iend-ip)); + size_t iSize = iSizeMax; + size_t const oSizeCand = (FUZ_rand(randState) & ((1<<nbBitsO)-1)) + 2; + size_t const oSizeMax = MIN(oSizeCand, (size_t)(oend-op)); + size_t oSize = oSizeMax; + BYTE const mark = (BYTE)(FUZ_rand(randState) & 255); + LZ4F_decompressOptions_t dOptions; + memset(&dOptions, 0, sizeof(dOptions)); + dOptions.stableDst = FUZ_rand(randState) & 1; + if (o_scenario == o_overwrite) dOptions.stableDst = 0; /* overwrite mode */ + if (op + oSizeMax < oend) op[oSizeMax] = mark; + + DISPLAYLEVEL(7, "dstCapacity=%u, presentedInput=%u \n", (unsigned)oSize, (unsigned)iSize); + + /* read data from byte-exact buffer to catch out-of-bound reads */ + { void* const iBuffer = malloc(iSizeMax); + assert(iBuffer != NULL); + memcpy(iBuffer, ip, iSizeMax); + moreToFlush = LZ4F_decompress(dCtx, op, &oSize, iBuffer, &iSize, &dOptions); + free(iBuffer); + } + DISPLAYLEVEL(7, "oSize=%u, readSize=%u \n", (unsigned)oSize, (unsigned)iSize); + + if (op + oSizeMax < oend) { + CHECK(op[oSizeMax] != mark, "op[oSizeMax] = %02X != %02X : " + "Decompression overwrites beyond assigned dst size", + op[oSizeMax], mark); + } + if (LZ4F_getErrorCode(moreToFlush) == LZ4F_ERROR_contentChecksum_invalid) + locateBuffDiff(srcRef, dst, decompressedSize, o_scenario); + if (LZ4F_isError(moreToFlush)) return moreToFlush; + + XXH64_update(&xxh64, op, oSize); + totalOut += oSize; + op += oSize; + ip += iSize; + if (o_scenario == o_noncontiguous) { + if (op == oend) return LZ4F_ERROR_GENERIC; /* can theoretically happen with bogus data */ + op++; /* create a gap between consecutive output */ + } + if (o_scenario==o_overwrite) op = (BYTE*)dst; /* overwrite destination */ + if ( (op == oend) /* no more room for output; can happen with bogus input */ + && (iSize == 0)) /* no input consumed */ + break; } + if (moreToFlush != 0) return LZ4F_ERROR_decompressionFailed; + if (totalOut) { /* otherwise, it's a skippable frame */ + U64 const crcDecoded = XXH64_digest(&xxh64); + if (crcDecoded != crcOrig) { + locateBuffDiff(srcRef, dst, decompressedSize, o_scenario); + return LZ4F_ERROR_contentChecksum_invalid; + } } + return 0; +} + + +size_t test_lz4f_decompression(const void* cSrc, size_t cSize, + const void* srcRef, size_t decompressedSize, + U64 crcOrig, + U32* const randState, + LZ4F_dctx* const dCtx, + U32 seed, U32 testNb) +{ + o_scenario_e const o_scenario = (o_scenario_e)(FUZ_rand(randState) % 3); /* 0 : contiguous; 1 : non-contiguous; 2 : dst overwritten */ + /* tighten dst buffer conditions */ + size_t const dstCapacity = (o_scenario == o_noncontiguous) ? + (decompressedSize * 2) + 128 : + decompressedSize; + size_t result; + void* const dstBuffer = malloc(dstCapacity); + assert(dstBuffer != NULL); + + result = test_lz4f_decompression_wBuffers(cSrc, cSize, + dstBuffer, dstCapacity, o_scenario, + srcRef, decompressedSize, + crcOrig, + randState, + dCtx, + seed, testNb); + + free(dstBuffer); + return result; } int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration_s) { - int testResult = 0; unsigned testNb = 0; - size_t const srcDataLength = 9 MB; /* needs to be > 2x4MB to test large blocks */ - void* srcBuffer = NULL; - size_t const compressedBufferSize = LZ4F_compressFrameBound(srcDataLength, NULL) + 4 MB; /* needs some margin */ + size_t const CNBufferLength = 9 MB; /* needs to be > 2x4MB to test large blocks */ + void* CNBuffer = NULL; + size_t const compressedBufferSize = LZ4F_compressFrameBound(CNBufferLength, NULL) + 4 MB; /* needs some margin */ void* compressedBuffer = NULL; void* decodedBuffer = NULL; U32 coreRand = seed; LZ4F_decompressionContext_t dCtx = NULL; + LZ4F_decompressionContext_t dCtxNoise = NULL; LZ4F_compressionContext_t cCtx = NULL; clock_t const startClock = clock(); clock_t const clockDuration = duration_s * CLOCKS_PER_SEC; -# undef CHECK -# define EXIT_MSG(...) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ - DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } -# define CHECK(cond, ...) { if (cond) { EXIT_MSG(__VA_ARGS__); } } /* Create buffers */ { size_t const creationStatus = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); } + { size_t const creationStatus = LZ4F_createDecompressionContext(&dCtxNoise, LZ4F_VERSION); + CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); } { size_t const creationStatus = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION); CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); } - srcBuffer = malloc(srcDataLength); - CHECK(srcBuffer==NULL, "srcBuffer Allocation failed"); + CNBuffer = malloc(CNBufferLength); + CHECK(CNBuffer==NULL, "CNBuffer Allocation failed"); compressedBuffer = malloc(compressedBufferSize); CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed"); - decodedBuffer = calloc(1, srcDataLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */ + decodedBuffer = calloc(1, CNBufferLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */ CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed"); - FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand); + FUZ_fillCompressibleNoiseBuffer(CNBuffer, CNBufferLength, compressibility, &coreRand); /* jump to requested testNb */ for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); /* sync randomizer */ @@ -819,10 +937,10 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi /* main fuzzer test loop */ for ( ; (testNb < nbTests) || (clockDuration > FUZ_GetClockSpan(startClock)) ; testNb++) { U32 randState = coreRand ^ prime1; - unsigned const srcBits = (FUZ_rand(&randState) % (FUZ_highbit((U32)(srcDataLength-1)) - 1)) + 1; + unsigned const srcBits = (FUZ_rand(&randState) % (FUZ_highbit((U32)(CNBufferLength-1)) - 1)) + 1; size_t const srcSize = (FUZ_rand(&randState) & ((1<<srcBits)-1)) + 1; - size_t const srcStartId = FUZ_rand(&randState) % (srcDataLength - srcSize); - const BYTE* const srcStart = (const BYTE*)srcBuffer + srcStartId; + size_t const srcStartId = FUZ_rand(&randState) % (CNBufferLength - srcSize); + const BYTE* const srcStart = (const BYTE*)CNBuffer + srcStartId; unsigned const neverFlush = (FUZ_rand(&randState) & 15) == 1; U64 const crcOrig = XXH64(srcStart, srcSize, 1); LZ4F_preferences_t prefs; @@ -848,9 +966,11 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi FUZ_writeLE32(op, LZ4F_MAGIC_SKIPPABLE_START + (FUZ_rand(&randState) & 15)); FUZ_writeLE32(op+4, (U32)srcSize); cSize = srcSize+8; + } else if ((FUZ_rand(&randState) & 0xF) == 2) { /* single pass compression (simple) */ cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(srcSize, prefsPtr), srcStart, srcSize, prefsPtr); CHECK(LZ4F_isError(cSize), "LZ4F_compressFrame failed : error %i (%s)", (int)cSize, LZ4F_getErrorName(cSize)); + } else { /* multi-segments compression */ const BYTE* ip = srcStart; const BYTE* const iend = srcStart + srcSize; @@ -913,58 +1033,53 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi DISPLAYLEVEL(5, "\nCompressed %u bytes into %u \n", (U32)srcSize, (U32)cSize); } + /* multi-segments decompression */ - { const BYTE* ip = (const BYTE*)compressedBuffer; - const BYTE* const iend = ip + cSize; - BYTE* op = (BYTE*)decodedBuffer; - BYTE* const oend = op + srcDataLength; - unsigned const suggestedBits = FUZ_highbit((U32)cSize); - unsigned const maxBits = MAX(3, suggestedBits); - unsigned const nonContiguousDst = FUZ_rand(&randState) % 3; /* 0 : contiguous; 1 : non-contiguous; 2 : dst overwritten */ - size_t totalOut = 0; - size_t decSize = 0; - XXH64_state_t xxh64; - XXH64_reset(&xxh64, 1); - assert(ip < iend); - while (ip < iend) { - unsigned const nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1; - unsigned const nbBitsO = (FUZ_rand(&randState) % (maxBits)) + 1; - size_t const iSizeMax = (FUZ_rand(&randState) & ((1<<nbBitsI)-1)) + 1; - size_t iSize = MIN(iSizeMax, (size_t)(iend-ip)); - size_t const oSizeMax = (FUZ_rand(&randState) & ((1<<nbBitsO)-1)) + 2; - size_t oSize = MIN(oSizeMax, (size_t)(oend-op)); - LZ4F_decompressOptions_t dOptions; - memset(&dOptions, 0, sizeof(dOptions)); - dOptions.stableDst = FUZ_rand(&randState) & 1; - if (nonContiguousDst==2) dOptions.stableDst = 0; /* overwrite mode */ - decSize = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions); - if (LZ4F_getErrorCode(decSize) == LZ4F_ERROR_contentChecksum_invalid) - locateBuffDiff(srcStart, decodedBuffer, srcSize, nonContiguousDst); - CHECK(LZ4F_isError(decSize), "Decompression failed (error %i:%s)", - (int)decSize, LZ4F_getErrorName(decSize)); - XXH64_update(&xxh64, op, (U32)oSize); - totalOut += oSize; - op += oSize; - ip += iSize; - op += nonContiguousDst; - if (nonContiguousDst==2) op = (BYTE*)decodedBuffer; /* overwritten destination */ - } - CHECK(decSize != 0, "Frame decompression failed (error %i)", (int)decSize); - if (totalOut) { /* otherwise, it's a skippable frame */ - U64 const crcDecoded = XXH64_digest(&xxh64); - if (crcDecoded != crcOrig) { - locateBuffDiff(srcStart, decodedBuffer, srcSize, nonContiguousDst); - EXIT_MSG("Decompression corruption"); - } } + DISPLAYLEVEL(6, "normal decompression \n"); + { size_t result = test_lz4f_decompression(compressedBuffer, cSize, srcStart, srcSize, crcOrig, &randState, dCtx, seed, testNb); + CHECK (LZ4F_isError(result), "multi-segment decompression failed (error %i => %s)", + (int)result, LZ4F_getErrorName(result)); } - } + +#if 1 + /* insert noise into src */ + { U32 const maxNbBits = FUZ_highbit((U32)cSize); + size_t pos = 0; + for (;;) { + /* keep some original src */ + { U32 const nbBits = FUZ_rand(&randState) % maxNbBits; + size_t const mask = (1<<nbBits) - 1; + size_t const skipLength = FUZ_rand(&randState) & mask; + pos += skipLength; + } + if (pos >= cSize) break; + /* add noise */ + { U32 const nbBitsCodes = FUZ_rand(&randState) % maxNbBits; + U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0; + size_t const mask = (1<<nbBits) - 1; + size_t const rNoiseLength = (FUZ_rand(&randState) & mask) + 1; + size_t const noiseLength = MIN(rNoiseLength, cSize-pos); + size_t const noiseStart = FUZ_rand(&randState) % (CNBufferLength - noiseLength); + memcpy((BYTE*)compressedBuffer + pos, (const char*)CNBuffer + noiseStart, noiseLength); + pos += noiseLength; + } } } + + /* test decompression on noisy src */ + DISPLAYLEVEL(6, "noisy decompression \n"); + test_lz4f_decompression(compressedBuffer, cSize, srcStart, srcSize, crcOrig, &randState, dCtxNoise, seed, testNb); + /* note : we don't analyze result here : it probably failed, which is expected. + * We just check for potential out-of-bound reads and writes. */ + LZ4F_resetDecompressionContext(dCtxNoise); /* context must be reset after an error */ +#endif + +} /* for ( ; (testNb < nbTests) ; ) */ DISPLAYLEVEL(2, "\rAll tests completed \n"); -_end: LZ4F_freeDecompressionContext(dCtx); + LZ4F_freeDecompressionContext(dCtxNoise); LZ4F_freeCompressionContext(cCtx); - free(srcBuffer); + free(CNBuffer); free(compressedBuffer); free(decodedBuffer); @@ -972,11 +1087,7 @@ _end: DISPLAY("press enter to finish \n"); (void)getchar(); } - return testResult; - -_output_error: - testResult = 1; - goto _end; + return 0; } @@ -1002,8 +1113,8 @@ int main(int argc, const char** argv) U32 seed=0; int seedset=0; int argNb; - int nbTests = nbTestsDefault; - int testNb = 0; + unsigned nbTests = nbTestsDefault; + unsigned testNb = 0; int proba = FUZ_COMPRESSIBILITY_DEFAULT; int result=0; U32 duration=0; @@ -1048,7 +1159,7 @@ int main(int argc, const char** argv) nbTests=0; duration=0; while ((*argument>='0') && (*argument<='9')) { nbTests *= 10; - nbTests += *argument - '0'; + nbTests += (unsigned)(*argument - '0'); argument++; } break; @@ -1071,7 +1182,7 @@ int main(int argc, const char** argv) case '6': case '7': case '8': - case '9': duration *= 10; duration += *argument++ - '0'; continue; + case '9': duration *= 10; duration += (U32)(*argument++ - '0'); continue; } break; } @@ -1083,7 +1194,7 @@ int main(int argc, const char** argv) seedset=1; while ((*argument>='0') && (*argument<='9')) { seed *= 10; - seed += *argument - '0'; + seed += (U32)(*argument - '0'); argument++; } break; @@ -1092,7 +1203,7 @@ int main(int argc, const char** argv) testNb=0; while ((*argument>='0') && (*argument<='9')) { testNb *= 10; - testNb += *argument - '0'; + testNb += (unsigned)(*argument - '0'); argument++; } break; @@ -1126,7 +1237,7 @@ int main(int argc, const char** argv) DISPLAY("Seed = %u\n", seed); if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba); - if (nbTests<=0) nbTests=1; + nbTests += (nbTests==0); /* avoid zero */ if (testNb==0) result = basicTests(seed, ((double)proba) / 100); if (result) return 1; diff --git a/tests/fullbench.c b/tests/fullbench.c index d2af662..7d74d3f 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -343,11 +343,44 @@ static int local_LZ4F_decompress(const char* in, char* out, int inSize, int outS assert(inSize >= 0); assert(outSize >= 0); result = LZ4F_decompress(g_dCtx, out, &dstSize, in, &srcSize, NULL); - if (result!=0) { DISPLAY("Error decompressing frame : unfinished frame\n"); exit(8); } - if (srcSize != (size_t)inSize) { DISPLAY("Error decompressing frame : read size incorrect\n"); exit(9); } + if (result!=0) { DISPLAY("Error decompressing frame : unfinished frame \n"); exit(8); } + if (srcSize != (size_t)inSize) { DISPLAY("Error decompressing frame : read size incorrect \n"); exit(9); } return (int)dstSize; } +static int local_LZ4F_decompress_followHint(const char* src, char* dst, int srcSize, int dstSize) +{ + size_t totalInSize = (size_t)srcSize; + size_t maxOutSize = (size_t)dstSize; + + size_t inPos = 0; + size_t inSize = 0; + size_t outPos = 0; + size_t outRemaining = maxOutSize - outPos; + + for (;;) { + size_t const sizeHint = LZ4F_decompress(g_dCtx, dst+outPos, &outRemaining, src+inPos, &inSize, NULL); + assert(!LZ4F_isError(sizeHint)); + + inPos += inSize; + inSize = sizeHint; + + outPos += outRemaining; + outRemaining = maxOutSize - outPos; + + if (!sizeHint) break; + } + + /* frame completed */ + if (inPos != totalInSize) { + DISPLAY("Error decompressing frame : must read (%u) full frame (%u) \n", + (unsigned)inPos, (unsigned)totalInSize); + exit(10); + } + return (int)outPos; + +} + #define NB_COMPRESSION_ALGORITHMS 100 #define NB_DECOMPRESSION_ALGORITHMS 100 @@ -446,7 +479,14 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) for (i=0; i<nbChunks; i++) { chunkP[i].id = (U32)i; chunkP[i].origBuffer = in; in += g_chunkSize; - if ((int)remaining > g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } + assert(g_chunkSize > 0); + if (remaining > (size_t)g_chunkSize) { + chunkP[i].origSize = g_chunkSize; + remaining -= (size_t)g_chunkSize; + } else { + chunkP[i].origSize = (int)remaining; + remaining = 0; + } chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; chunkP[i].compressedSize = 0; } @@ -501,9 +541,10 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) if (initFunction!=NULL) initFunction(); for (chunkNb=0; chunkNb<nbChunks; chunkNb++) { chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize); - if (chunkP[chunkNb].compressedSize==0) - DISPLAY("ERROR ! %s() = 0 !! \n", compressorName), exit(1); - } + if (chunkP[chunkNb].compressedSize==0) { + DISPLAY("ERROR ! %s() = 0 !! \n", compressorName); + exit(1); + } } nb_loops++; } clockTime = BMK_GetClockSpan(clockTime); @@ -511,7 +552,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) nb_loops += !nb_loops; /* avoid division by zero */ averageTime = ((double)clockTime) / nb_loops / CLOCKS_PER_SEC; if (averageTime < bestTime) bestTime = averageTime; - cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize; + cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += (size_t)chunkP[chunkNb].compressedSize; ratio = (double)cSize/(double)benchedSize*100.; PROGRESS("%2i-%-34.34s :%10i ->%9i (%5.2f%%),%7.1f MB/s\r", loopNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000000); } @@ -531,23 +572,30 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) nbChunks = (int) (((int)benchedSize + (g_chunkSize-1))/ g_chunkSize); for (i=0; i<nbChunks; i++) { - chunkP[i].id = i; + chunkP[i].id = (U32)i; chunkP[i].origBuffer = in; in += g_chunkSize; - if ((int)remaining > g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } + if ((int)remaining > g_chunkSize) { + chunkP[i].origSize = g_chunkSize; + remaining -= (size_t)g_chunkSize; + } else { + chunkP[i].origSize = (int)remaining; + remaining = 0; + } chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; chunkP[i].compressedSize = 0; } } for (chunkNb=0; chunkNb<nbChunks; chunkNb++) { chunkP[chunkNb].compressedSize = LZ4_compress_default(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize, maxCompressedChunkSize); - if (chunkP[chunkNb].compressedSize==0) - DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress"), exit(1); - } + if (chunkP[chunkNb].compressedSize==0) { + DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress"); + exit(1); + } } /* Decompression Algorithms */ for (dAlgNb=0; (dAlgNb <= NB_DECOMPRESSION_ALGORITHMS) && g_decompressionTest; dAlgNb++) { - const char* dName; - int (*decompressionFunction)(const char*, char*, int, int); + const char* dName = NULL; + int (*decompressionFunction)(const char*, char*, int, int) = NULL; double bestTime = 100000000.; int checkResult = 1; @@ -565,17 +613,15 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) #ifndef LZ4_DLL_IMPORT case 8: decompressionFunction = local_LZ4_decompress_safe_forceExtDict; dName = "LZ4_decompress_safe_forceExtDict"; break; #endif - case 9: decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress"; - { size_t const errorCode = LZ4F_compressFrame(compressed_buff, compressedBuffSize, orig_buff, benchedSize, NULL); - if (LZ4F_isError(errorCode)) { - DISPLAY("Error while preparing compressed frame\n"); - free(orig_buff); - free(compressed_buff); - free(chunkP); - return 1; - } + case 10: + case 11: + if (dAlgNb == 10) { decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress"; } /* can be skipped */ + if (dAlgNb == 11) { decompressionFunction = local_LZ4F_decompress_followHint; dName = "LZ4F_decompress_followHint"; } /* can be skipped */ + /* prepare compressed data using frame format */ + { size_t const fcsize = LZ4F_compressFrame(compressed_buff, (size_t)compressedBuffSize, orig_buff, benchedSize, NULL); + assert(!LZ4F_isError(fcsize)); chunkP[0].origSize = (int)benchedSize; - chunkP[0].compressedSize = (int)errorCode; + chunkP[0].compressedSize = (int)fcsize; nbChunks = 1; break; } @@ -583,6 +629,9 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) continue; /* skip if unknown ID */ } + assert(decompressionFunction != NULL); + assert(dName != NULL); + { size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } /* zeroing source area, for CRC checking */ for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) { diff --git a/tests/fuzzer.c b/tests/fuzzer.c index a5b5c93..8a095c4 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -122,6 +122,14 @@ static U32 FUZ_rotl32(U32 u32, U32 nbBits) return ((u32 << nbBits) | (u32 >> (32 - nbBits))); } +static U32 FUZ_highbit32(U32 v32) +{ + unsigned nbBits = 0; + if (v32==0) return 0; + while (v32) { v32 >>= 1; nbBits++; } + return nbBits; +} + static U32 FUZ_rand(U32* src) { U32 rand32 = *src; @@ -207,7 +215,7 @@ static int FUZ_AddressOverflow(void) } { size_t const sizeToGenerateOverflow = (size_t)(- ((uintptr_t)buffers[nbBuff-1]) + 512); - unsigned const nbOf255 = (unsigned)((sizeToGenerateOverflow / 255) + 1); + int const nbOf255 = (int)((sizeToGenerateOverflow / 255) + 1); char* const input = buffers[nbBuff-1]; char* output = buffers[nbBuff]; int r; @@ -215,7 +223,7 @@ static int FUZ_AddressOverflow(void) input[1] = (char)0xFF; input[2] = (char)0xFF; input[3] = (char)0xFF; - { unsigned u; for(u = 4; u <= nbOf255+4; u++) input[u] = (char)0xff; } + { int u; for(u = 4; u <= nbOf255+4; u++) input[u] = (char)0xff; } r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134); if (r>0) { DISPLAY("LZ4_decompress_safe = %i \n", r); goto _overflowError; } input[0] = (char)0x1F; /* Match length overflow */ @@ -366,7 +374,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c U32 testNb = 0; U32 randState = FUZ_rand(&coreRandState) ^ PRIME3; int const blockSize = (FUZ_rand(&randState) % (FUZ_MAX_BLOCK_SIZE-1)) + 1; - int const blockStart = (FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize - 1)) + 1; + int const blockStart = (int)(FUZ_rand(&randState) % (U32)(COMPRESSIBLE_NOISE_LENGTH - blockSize - 1)) + 1; int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; int const dictSize = MIN(dictSizeRand, blockStart - 1); int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1); @@ -388,69 +396,65 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Test compression destSize */ FUZ_DISPLAYTEST("test LZ4_compress_destSize()"); - { int srcSize = blockSize; - int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7; - char endCheck = FUZ_rand(&randState) & 255; + { int cSize, srcSize = blockSize; + int const targetSize = srcSize * (int)((FUZ_rand(&randState) & 127)+1) >> 7; + char const endCheck = (char)(FUZ_rand(&randState) & 255); compressedBuffer[targetSize] = endCheck; - ret = LZ4_compress_destSize(block, compressedBuffer, &srcSize, targetSize); - FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_destSize() result larger than dst buffer !"); + cSize = LZ4_compress_destSize(block, compressedBuffer, &srcSize, targetSize); + FUZ_CHECKTEST(cSize > targetSize, "LZ4_compress_destSize() result larger than dst buffer !"); FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_destSize() overwrite dst buffer !"); - FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_destSize() fed more than src buffer !"); - DISPLAYLEVEL(5, "destSize : %7i/%7i; content%7i/%7i ", ret, targetSize, srcSize, blockSize); + FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_destSize() read more than src buffer !"); + DISPLAYLEVEL(5, "destSize : %7i/%7i; content%7i/%7i ", cSize, targetSize, srcSize, blockSize); if (targetSize>0) { /* check correctness */ U32 const crcBase = XXH32(block, (size_t)srcSize, 0); - char const canary = FUZ_rand(&randState) & 255; - FUZ_CHECKTEST((ret==0), "LZ4_compress_destSize() compression failed"); + char const canary = (char)(FUZ_rand(&randState) & 255); + FUZ_CHECKTEST((cSize==0), "LZ4_compress_destSize() compression failed"); FUZ_DISPLAYTEST(); - compressedSize = ret; decodedBuffer[srcSize] = canary; - ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compress_destSize"); - FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data"); + { int const dSize = LZ4_decompress_safe(compressedBuffer, decodedBuffer, cSize, srcSize); + FUZ_CHECKTEST(dSize<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compress_destSize"); + FUZ_CHECKTEST(dSize!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data"); + } FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe() overwrite dst buffer !"); - { U32 const crcDec = XXH32(decodedBuffer, srcSize, 0); - FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); } - - DISPLAYLEVEL(5, " OK \n"); - } else { - DISPLAYLEVEL(5, " \n"); - } } + { U32 const crcDec = XXH32(decodedBuffer, (size_t)srcSize, 0); + FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); + } } + DISPLAYLEVEL(5, " OK \n"); + } /* Test compression HC destSize */ FUZ_DISPLAYTEST("test LZ4_compress_HC_destSize()"); - { int srcSize = blockSize; - int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7; - char const endCheck = FUZ_rand(&randState) & 255; - void* ctx = LZ4_createHC(block); + { int cSize, srcSize = blockSize; + int const targetSize = srcSize * (int)((FUZ_rand(&randState) & 127)+1) >> 7; + char const endCheck = (char)(FUZ_rand(&randState) & 255); + void* const ctx = LZ4_createHC(block); FUZ_CHECKTEST(ctx==NULL, "LZ4_createHC() allocation failed"); compressedBuffer[targetSize] = endCheck; - ret = LZ4_compress_HC_destSize(ctx, block, compressedBuffer, &srcSize, targetSize, compressionLevel); + cSize = LZ4_compress_HC_destSize(ctx, block, compressedBuffer, &srcSize, targetSize, compressionLevel); DISPLAYLEVEL(5, "LZ4_compress_HC_destSize(%i): destSize : %7i/%7i; content%7i/%7i ", - compressionLevel, ret, targetSize, srcSize, blockSize); + compressionLevel, cSize, targetSize, srcSize, blockSize); LZ4_freeHC(ctx); - FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_HC_destSize() result larger than dst buffer !"); + FUZ_CHECKTEST(cSize > targetSize, "LZ4_compress_HC_destSize() result larger than dst buffer !"); FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_HC_destSize() overwrite dst buffer !"); FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_HC_destSize() fed more than src buffer !"); if (targetSize>0) { /* check correctness */ U32 const crcBase = XXH32(block, (size_t)srcSize, 0); - char const canary = FUZ_rand(&randState) & 255; - FUZ_CHECKTEST((ret==0), "LZ4_compress_HC_destSize() compression failed"); + char const canary = (char)(FUZ_rand(&randState) & 255); + FUZ_CHECKTEST((cSize==0), "LZ4_compress_HC_destSize() compression failed"); FUZ_DISPLAYTEST(); - compressedSize = ret; decodedBuffer[srcSize] = canary; - ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compressHC_destSize"); - FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data"); + { int const dSize = LZ4_decompress_safe(compressedBuffer, decodedBuffer, cSize, srcSize); + FUZ_CHECKTEST(dSize<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compressHC_destSize"); + FUZ_CHECKTEST(dSize!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data"); + } FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe() overwrite dst buffer !"); - { U32 const crcDec = XXH32(decodedBuffer, srcSize, 0); + { U32 const crcDec = XXH32(decodedBuffer, (size_t)srcSize, 0); FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); - } - DISPLAYLEVEL(5, " OK \n"); - } else { - DISPLAYLEVEL(5, " \n"); - } } + } } + DISPLAYLEVEL(5, " OK \n"); + } /* Test compression HC */ FUZ_DISPLAYTEST("test LZ4_compress_HC()"); @@ -524,7 +528,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(r!=blockSize, "LZ4_decompress_safe did not regenerate original data"); } FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size"); - { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); } @@ -559,13 +563,49 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe overrun specified output buffer size"); } + /* noisy src decompression test */ + + /* insert noise into src */ + { U32 const maxNbBits = FUZ_highbit32((U32)compressedSize); + size_t pos = 0; + for (;;) { + /* keep some original src */ + { U32 const nbBits = FUZ_rand(&randState) % maxNbBits; + size_t const mask = (1<<nbBits) - 1; + size_t const skipLength = FUZ_rand(&randState) & mask; + pos += skipLength; + } + if (pos >= (size_t)compressedSize) break; + /* add noise */ + { U32 const nbBitsCodes = FUZ_rand(&randState) % maxNbBits; + U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0; + size_t const mask = (1<<nbBits) - 1; + size_t const rNoiseLength = (FUZ_rand(&randState) & mask) + 1; + size_t const noiseLength = MIN(rNoiseLength, (size_t)compressedSize-pos); + size_t const noiseStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - noiseLength); + memcpy(cBuffer_exact + pos, (const char*)CNBuffer + noiseStart, noiseLength); + pos += noiseLength; + } } } + + /* decompress noisy source */ + FUZ_DISPLAYTEST("decompress noisy source "); + { U32 const endMark = 0xA9B1C3D6; + memcpy(decodedBuffer+blockSize, &endMark, sizeof(endMark)); + { int const decompressResult = LZ4_decompress_safe(cBuffer_exact, decodedBuffer, compressedSize, blockSize); + /* result *may* be an unlikely success, but even then, it must strictly respect dst buffer boundaries */ + FUZ_CHECKTEST(decompressResult > blockSize, "LZ4_decompress_safe on noisy src : result is too large : %u > %u (dst buffer)", (unsigned)decompressResult, (unsigned)blockSize); + } + { U32 endCheck; memcpy(&endCheck, decodedBuffer+blockSize, sizeof(endCheck)); + FUZ_CHECKTEST(endMark!=endCheck, "LZ4_decompress_safe on noisy src : dst buffer overflow"); + } } /* noisy src decompression test */ + free(cBuffer_exact); } /* Test decoding with input size being one byte too short => must fail */ FUZ_DISPLAYTEST(); { int const r = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize); - FUZ_CHECKTEST(r>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize); + FUZ_CHECKTEST(r>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, result=%i, compressedSize=%i)", blockSize, r, compressedSize); } /* Test decoding with input size being one byte too large => must fail */ @@ -578,7 +618,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Test partial decoding => must work */ FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial"); - { size_t const missingBytes = FUZ_rand(&randState) % blockSize; + { size_t const missingBytes = FUZ_rand(&randState) % (unsigned)blockSize; int const targetSize = (int)((size_t)blockSize - missingBytes); char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A; int const decResult = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, targetSize, blockSize); @@ -615,8 +655,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c if (missingBytes >= compressedSize) missingBytes = compressedSize-1; missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ compressedBuffer[compressedSize-missingBytes] = 0; - ret = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize-missingBytes); - FUZ_CHECKTEST(ret, "LZ4_compress_default should have failed (output buffer too small by %i byte)", missingBytes); + { int const cSize = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize-missingBytes); + FUZ_CHECKTEST(cSize, "LZ4_compress_default should have failed (output buffer too small by %i byte)", missingBytes); + } FUZ_CHECKTEST(compressedBuffer[compressedSize-missingBytes], "LZ4_compress_default overran output buffer ! (%i missingBytes)", missingBytes) } @@ -626,8 +667,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c if (missingBytes >= HCcompressedSize) missingBytes = HCcompressedSize-1; missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ compressedBuffer[HCcompressedSize-missingBytes] = 0; - ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, compressionLevel); - FUZ_CHECKTEST(ret, "LZ4_compress_HC should have failed (output buffer too small by %i byte)", missingBytes); + { int const hcSize = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, compressionLevel); + FUZ_CHECKTEST(hcSize, "LZ4_compress_HC should have failed (output buffer too small by %i byte)", missingBytes); + } FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-missingBytes], "LZ4_compress_HC overran output buffer ! (%i missingBytes)", missingBytes) } @@ -654,8 +696,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c if (crcCheck!=crcOrig) { FUZ_findDiff(block, decodedBuffer); EXIT_MSG("LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); - } - } + } } FUZ_DISPLAYTEST("test LZ4_decompress_safe_usingDict()"); ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize); @@ -672,7 +713,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue failed"); - FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary but with an output buffer too short by one byte"); + FUZ_DISPLAYTEST("LZ4_compress_fast_continue() with dictionary and output buffer too short by one byte"); LZ4_loadDict(&LZ4dictBody, dict, dictSize); ret = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1); FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize); @@ -697,15 +738,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c if (crcCheck!=crcOrig) { FUZ_findDiff(block, decodedBuffer); EXIT_MSG("LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); - } - } + } } FUZ_DISPLAYTEST(); decodedBuffer[blockSize] = 0; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); - { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); } @@ -722,17 +762,16 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); FUZ_DISPLAYTEST(); - { U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2; - if ((U32)blockSize > missingBytes) { + { int const missingBytes = (FUZ_rand(&randState) & 0xF) + 2; + if (blockSize > missingBytes) { decodedBuffer[blockSize-missingBytes] = 0; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-missingBytes, dict, dictSize); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%u byte)", missingBytes); - FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize); + FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%i byte)", missingBytes); + FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%i byte) (blockSize=%i)", missingBytes, blockSize); } } /* Compress using external dictionary stream */ - { - LZ4_stream_t LZ4_stream; + { LZ4_stream_t LZ4_stream; int expectedSize; U32 expectedCrc; @@ -740,7 +779,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c LZ4_loadDict(&LZ4dictBody, dict, dictSize); expectedSize = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1); FUZ_CHECKTEST(expectedSize<=0, "LZ4_compress_fast_continue reference compression for extDictCtx should have succeeded"); - expectedCrc = XXH32(compressedBuffer, expectedSize, 0); + expectedCrc = XXH32(compressedBuffer, (size_t)expectedSize, 0); FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary()"); LZ4_loadDict(&LZ4dictBody, dict, dictSize); @@ -756,7 +795,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c * test. */ FUZ_CHECKTEST(blockContinueCompressedSize != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output (%d expected vs %d actual)", expectedSize, blockContinueCompressedSize); - FUZ_CHECKTEST(XXH32(compressedBuffer, blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output"); + FUZ_CHECKTEST(XXH32(compressedBuffer, (size_t)blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output"); FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary(), but output buffer is 1 byte too short"); LZ4_resetStream_fast(&LZ4_stream); @@ -772,7 +811,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx should work : enough size available within output buffer"); FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output"); - FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output"); + FUZ_CHECKTEST(XXH32(compressedBuffer, (size_t)ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output"); FUZ_CHECKTEST(LZ4_stream.internal_donotuse.dirty, "context should be good"); FUZ_DISPLAYTEST(); @@ -782,7 +821,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx with re-used context should work : enough size available within output buffer"); FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output"); - FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output"); + FUZ_CHECKTEST(XXH32(compressedBuffer, (size_t)ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output"); FUZ_CHECKTEST(LZ4_stream.internal_donotuse.dirty, "context should be good"); } @@ -792,12 +831,11 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size"); - { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0); if (crcCheck!=crcOrig) { FUZ_findDiff(block, decodedBuffer); EXIT_MSG("LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); - } - } + } } FUZ_DISPLAYTEST(); decodedBuffer[blockSize] = 0; @@ -823,7 +861,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_DISPLAYTEST("LZ4_decompress_safe_usingDict with a too small output buffer"); { U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2; if ((U32)blockSize > missingBytes) { - decodedBuffer[blockSize-missingBytes] = 0; + decodedBuffer[(U32)blockSize-missingBytes] = 0; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-missingBytes, dict, dictSize); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%u byte)", missingBytes); FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize); @@ -965,8 +1003,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c } -#define testInputSize (192 KB) -#define testCompressedSize (128 KB) +#define testInputSize (196 KB) +#define testCompressedSize (130 KB) #define ringBufferSize (8 KB) static void FUZ_unitTests(int compressionLevel) @@ -990,17 +1028,17 @@ static void FUZ_unitTests(int compressionLevel) FUZ_AddressOverflow(); /* Test decoding with empty input */ - DISPLAYLEVEL(3, "LZ4_decompress_safe() with empty input"); + DISPLAYLEVEL(3, "LZ4_decompress_safe() with empty input \n"); LZ4_decompress_safe(testCompressed, testVerify, 0, testInputSize); /* Test decoding with a one byte input */ - DISPLAYLEVEL(3, "LZ4_decompress_safe() with one byte input"); + DISPLAYLEVEL(3, "LZ4_decompress_safe() with one byte input \n"); { char const tmp = (char)0xFF; LZ4_decompress_safe(&tmp, testVerify, 1, testInputSize); } /* Test decoding shortcut edge case */ - DISPLAYLEVEL(3, "LZ4_decompress_safe() with shortcut edge case"); + DISPLAYLEVEL(3, "LZ4_decompress_safe() with shortcut edge case \n"); { char tmp[17]; /* 14 bytes of literals, followed by a 14 byte match. * Should not read beyond the end of the buffer. @@ -1013,6 +1051,54 @@ static void FUZ_unitTests(int compressionLevel) FUZ_CHECKTEST(r >= 0, "LZ4_decompress_safe() should fail"); } } + /* in-place compression test */ + DISPLAYLEVEL(3, "in-place compression using LZ4_compress_default() :"); + { int const sampleSize = 65 KB; + int const maxCSize = LZ4_COMPRESSBOUND(sampleSize); + int const outSize = LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCSize); + int const startInputIndex = outSize - sampleSize; + char* const startInput = testCompressed + startInputIndex; + XXH32_hash_t const crcOrig = XXH32(testInput, sampleSize, 0); + int cSize; + assert(outSize < (int)testCompressedSize); + memcpy(startInput, testInput, sampleSize); /* copy at end of buffer */ + /* compress in-place */ + cSize = LZ4_compress_default(startInput, testCompressed, sampleSize, maxCSize); + assert(cSize != 0); /* ensure compression is successful */ + assert(maxCSize < INT_MAX); + assert(cSize <= maxCSize); + /* decompress and verify */ + { int const dSize = LZ4_decompress_safe(testCompressed, testVerify, cSize, testInputSize); + assert(dSize == sampleSize); /* correct size */ + { XXH32_hash_t const crcCheck = XXH32(testVerify, (size_t)dSize, 0); + assert(crcCheck == crcOrig); + } } } + DISPLAYLEVEL(3, " OK \n"); + + /* in-place decompression test */ + DISPLAYLEVEL(3, "in-place decompression, limit case:"); + { int const sampleSize = 65 KB; + + FUZ_fillCompressibleNoiseBuffer(testInput, sampleSize, 0.0, &randState); + memset(testInput, 0, 267); /* calculated exactly so that compressedSize == originalSize-1 */ + + { XXH64_hash_t const crcOrig = XXH64(testInput, sampleSize, 0); + int const cSize = LZ4_compress_default(testInput, testCompressed, sampleSize, testCompressedSize); + assert(cSize == sampleSize - 1); /* worst case for in-place decompression */ + + { int const bufferSize = LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(sampleSize); + int const startInputIndex = bufferSize - cSize; + char* const startInput = testVerify + startInputIndex; + memcpy(startInput, testCompressed, cSize); + + /* decompress and verify */ + { int const dSize = LZ4_decompress_safe(startInput, testVerify, cSize, sampleSize); + assert(dSize == sampleSize); /* correct size */ + { XXH64_hash_t const crcCheck = XXH64(testVerify, (size_t)dSize, 0); + assert(crcCheck == crcOrig); + } } } } } + DISPLAYLEVEL(3, " OK \n"); + /* LZ4 streaming tests */ { LZ4_stream_t streamingState; U64 crcOrig; @@ -1061,17 +1147,17 @@ static void FUZ_unitTests(int compressionLevel) crcOrig = XXH64_digest(&xxhOrig); memcpy (ringBuffer + rNext, testInput + iNext, messageSize); - compressedSize = LZ4_compress_fast_continue(&streamingState, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize, 1); + compressedSize = LZ4_compress_fast_continue(&streamingState, ringBuffer + rNext, testCompressed, (int)messageSize, testCompressedSize-ringBufferSize, 1); FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_fast_continue() compression failed"); - result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, messageSize); + result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, (int)messageSize); FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe_continue() test failed"); XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize); { U64 const crcNew = XXH64_digest(&xxhNewSafe); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); } - result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, messageSize); + result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, (int)messageSize); FUZ_CHECKTEST(result!=compressedSize, "ringBuffer : LZ4_decompress_fast_continue() test failed"); XXH64_update(&xxhNewFast, testVerify + dNext, messageSize); @@ -1151,12 +1237,12 @@ static void FUZ_unitTests(int compressionLevel) { U64 const crc64 = XXH64(testInput + 64 KB, testCompressedSize, 0); LZ4_resetStreamHC_fast(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, 64 KB); - result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); - FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); - - result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 64 KB); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed"); + { int const cSize = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); + FUZ_CHECKTEST(cSize==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : @return = %i", cSize); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); + { int const dSize = LZ4_decompress_safe_usingDict(testCompressed, testVerify, cSize, testCompressedSize, testInput, 64 KB); + FUZ_CHECKTEST(dSize!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed"); + } } { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() simple dictionary decompression test : corruption"); } } @@ -1165,7 +1251,8 @@ static void FUZ_unitTests(int compressionLevel) /* multiple HC compression test with dictionary */ { int result1, result2; int segSize = testCompressedSize / 2; - U64 const crc64 = XXH64(testInput + segSize, testCompressedSize, 0); + XXH64_hash_t const crc64 = ( (void)assert((unsigned)segSize + testCompressedSize < testInputSize) , + XXH64(testInput + segSize, testCompressedSize, 0) ); LZ4_resetStreamHC_fast(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, segSize); result1 = LZ4_compress_HC_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1); @@ -1179,7 +1266,7 @@ static void FUZ_unitTests(int compressionLevel) FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 1 failed"); result = LZ4_decompress_safe_usingDict(testCompressed+result1, testVerify+segSize, result2, segSize, testInput, 2*segSize); FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 2 failed"); - { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); + { XXH64_hash_t const crcNew = XXH64(testVerify, testCompressedSize, 0); FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); } } @@ -1201,15 +1288,15 @@ static void FUZ_unitTests(int compressionLevel) { XXH64_state_t crcOrigState; XXH64_state_t crcNewState; const char* dict = testInput + 3; - int dictSize = (FUZ_rand(&randState) & 8191); + size_t dictSize = (FUZ_rand(&randState) & 8191); char* dst = testVerify; - size_t segStart = (size_t)dictSize + 7; - int segSize = (FUZ_rand(&randState) & 8191); + size_t segStart = dictSize + 7; + size_t segSize = (FUZ_rand(&randState) & 8191); int segNb = 1; LZ4_resetStreamHC_fast(&sHC, compressionLevel); - LZ4_loadDictHC(&sHC, dict, dictSize); + LZ4_loadDictHC(&sHC, dict, (int)dictSize); XXH64_reset(&crcOrigState, 0); XXH64_reset(&crcNewState, 0); @@ -1217,29 +1304,28 @@ static void FUZ_unitTests(int compressionLevel) while (segStart + segSize < testInputSize) { XXH64_update(&crcOrigState, testInput + segStart, segSize); crcOrig = XXH64_digest(&crcOrigState); - result = LZ4_compress_HC_continue(&sHC, testInput + segStart, testCompressed, segSize, LZ4_compressBound(segSize)); + assert(segSize <= INT_MAX); + result = LZ4_compress_HC_continue(&sHC, testInput + segStart, testCompressed, (int)segSize, LZ4_compressBound((int)segSize)); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); - result = LZ4_decompress_safe_usingDict(testCompressed, dst, result, segSize, dict, dictSize); - FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %i failed", segNb); + result = LZ4_decompress_safe_usingDict(testCompressed, dst, result, (int)segSize, dict, (int)dictSize); + FUZ_CHECKTEST(result!=(int)segSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %i failed", (int)segNb); XXH64_update(&crcNewState, dst, segSize); { U64 const crcNew = XXH64_digest(&crcNewState); if (crcOrig != crcNew) FUZ_findDiff(dst, testInput+segStart); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() part %i corruption", segNb); } - assert(segSize >= 0); dict = dst; dictSize = segSize; - dst += (size_t)segSize + 1; + dst += segSize + 1; segNb ++; - segStart += (size_t)segSize + (FUZ_rand(&randState) & 0xF) + 1; + segStart += segSize + (FUZ_rand(&randState) & 0xF) + 1; segSize = (FUZ_rand(&randState) & 8191); - } - } + } } /* ring buffer test */ { XXH64_state_t xxhOrig; @@ -1266,18 +1352,21 @@ static void FUZ_unitTests(int compressionLevel) crcOrig = XXH64_digest(&xxhOrig); memcpy (ringBuffer + rNext, testInput + iNext, messageSize); - compressedSize = LZ4_compress_HC_continue(&sHC, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); + assert(messageSize < INT_MAX); + compressedSize = LZ4_compress_HC_continue(&sHC, ringBuffer + rNext, testCompressed, (int)messageSize, testCompressedSize-ringBufferSize); FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed"); FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); - result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, messageSize); + assert(messageSize < INT_MAX); + result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, (int)messageSize); FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe_continue() test failed"); XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize); { U64 const crcNew = XXH64_digest(&xxhNewSafe); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); } - result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, messageSize); + assert(messageSize < INT_MAX); + result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, (int)messageSize); FUZ_CHECKTEST(result!=compressedSize, "ringBuffer : LZ4_decompress_fast_continue() test failed"); XXH64_update(&xxhNewFast, testVerify + dNext, messageSize); @@ -1337,14 +1426,14 @@ static void FUZ_unitTests(int compressionLevel) result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, ringBufferSafe + dNext, compressedSize, messageSize); FUZ_CHECKTEST(result!=messageSize, "64K D.ringBuffer : LZ4_decompress_safe_continue() test failed"); - XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, messageSize); + XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, (size_t)messageSize); { U64 const crcNew = XXH64_digest(&xxhNewSafe); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); } result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, ringBufferFast + dNext, messageSize); FUZ_CHECKTEST(result!=compressedSize, "64K D.ringBuffer : LZ4_decompress_fast_continue() test failed"); - XXH64_update(&xxhNewFast, ringBufferFast + dNext, messageSize); + XXH64_update(&xxhNewFast, ringBufferFast + dNext, (size_t)messageSize); { U64 const crcNew = XXH64_digest(&xxhNewFast); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption"); } @@ -1360,7 +1449,7 @@ static void FUZ_unitTests(int compressionLevel) dNext = 0; while (totalMessageSize < 9 MB) { - XXH64_update(&xxhOrig, testInput + iNext, messageSize); + XXH64_update(&xxhOrig, testInput + iNext, (size_t)messageSize); crcOrig = XXH64_digest(&xxhOrig); compressedSize = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); @@ -1375,7 +1464,7 @@ static void FUZ_unitTests(int compressionLevel) testCompressed, ringBufferSafe + dNext, compressedSize, dBufferSize - dNext); /* works without knowing messageSize, under assumption that messageSize <= maxMessageSize */ FUZ_CHECKTEST(result!=messageSize, "D.ringBuffer : LZ4_decompress_safe_continue() test failed"); - XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, messageSize); + XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, (size_t)messageSize); { U64 const crcNew = XXH64_digest(&xxhNewSafe); if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, ringBufferSafe + dNext); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption during D.ringBuffer test"); @@ -1384,7 +1473,7 @@ static void FUZ_unitTests(int compressionLevel) /* test LZ4_decompress_fast_continue in its own buffer ringBufferFast */ result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, ringBufferFast + dNext, messageSize); FUZ_CHECKTEST(result!=compressedSize, "D.ringBuffer : LZ4_decompress_fast_continue() test failed"); - XXH64_update(&xxhNewFast, ringBufferFast + dNext, messageSize); + XXH64_update(&xxhNewFast, ringBufferFast + dNext, (size_t)messageSize); { U64 const crcNew = XXH64_digest(&xxhNewFast); if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, ringBufferFast + dNext); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption during D.ringBuffer test"); @@ -1392,7 +1481,8 @@ static void FUZ_unitTests(int compressionLevel) /* prepare next message */ dNext += messageSize; - totalMessageSize += messageSize; + assert(messageSize >= 0); + totalMessageSize += (unsigned)messageSize; messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1; iNext = (FUZ_rand(&randState) & 65535); if (dNext + maxMessageSize > dBufferSize) dNext = 0; @@ -1410,6 +1500,11 @@ static void FUZ_unitTests(int compressionLevel) } + +/* ======================================= + * CLI + * ======================================= */ + static int FUZ_usage(const char* programName) { DISPLAY( "Usage :\n"); @@ -1433,8 +1528,8 @@ int main(int argc, const char** argv) U32 seed = 0; int seedset = 0; int argNb; - int nbTests = NB_ATTEMPTS; - int testNb = 0; + unsigned nbTests = NB_ATTEMPTS; + unsigned testNb = 0; int proba = FUZ_COMPRESSIBILITY_DEFAULT; int use_pause = 0; const char* programName = argv[0]; @@ -1472,7 +1567,7 @@ int main(int argc, const char** argv) nbTests = 0; duration = 0; while ((*argument>='0') && (*argument<='9')) { nbTests *= 10; - nbTests += *argument - '0'; + nbTests += (unsigned)(*argument - '0'); argument++; } break; @@ -1495,7 +1590,7 @@ int main(int argc, const char** argv) case '6': case '7': case '8': - case '9': duration *= 10; duration += *argument++ - '0'; continue; + case '9': duration *= 10; duration += (U32)(*argument++ - '0'); continue; } break; } @@ -1506,7 +1601,7 @@ int main(int argc, const char** argv) seed=0; seedset=1; while ((*argument>='0') && (*argument<='9')) { seed *= 10; - seed += *argument - '0'; + seed += (U32)(*argument - '0'); argument++; } break; @@ -1516,7 +1611,7 @@ int main(int argc, const char** argv) testNb=0; while ((*argument>='0') && (*argument<='9')) { testNb *= 10; - testNb += *argument - '0'; + testNb += (unsigned)(*argument - '0'); argument++; } break; @@ -1551,7 +1646,7 @@ int main(int argc, const char** argv) if ((seedset==0) && (testNb==0)) { FUZ_unitTests(LZ4HC_CLEVEL_DEFAULT); FUZ_unitTests(LZ4HC_CLEVEL_OPT_MIN); } - if (nbTests<=0) nbTests=1; + nbTests += (nbTests==0); /* avoid zero */ { int const result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100, duration); if (use_pause) { diff --git a/tests/test-lz4-list.py b/tests/test-lz4-list.py new file mode 100644 index 0000000..ce89757 --- /dev/null +++ b/tests/test-lz4-list.py @@ -0,0 +1,282 @@ +#! /usr/bin/env python3 +import subprocess +import time +import glob +import os +import tempfile +import unittest + +SIZES = [3, 11] # Always 2 sizes +MIB = 1048576 +LZ4 = os.path.dirname(os.path.realpath(__file__)) + "/../lz4" +if not os.path.exists(LZ4): + LZ4 = os.path.dirname(os.path.realpath(__file__)) + "/../programs/lz4" +TEMP = tempfile.gettempdir() + + +class NVerboseFileInfo(object): + def __init__(self, line_in): + self.line = line_in + splitlines = line_in.split() + if len(splitlines) != 7: + errout("Unexpected line: {}".format(line_in)) + self.frames, self.type, self.block, self.compressed, self.uncompressed, self.ratio, self.filename = splitlines + self.exp_unc_size = 0 + # Get real file sizes + if "concat-all" in self.filename or "2f--content-size" in self.filename: + for i in SIZES: + self.exp_unc_size += os.path.getsize("{}/test_list_{}M".format(TEMP, i)) + else: + uncompressed_filename = self.filename.split("-")[0] + self.exp_unc_size += os.path.getsize("{}/{}".format(TEMP, uncompressed_filename)) + self.exp_comp_size = os.path.getsize("{}/{}".format(TEMP, self.filename)) + + +class TestNonVerbose(unittest.TestCase): + @classmethod + def setUpClass(self): + self.nvinfo_list = [] + for i, line in enumerate(execute("{} --list -m {}/test_list_*.lz4".format(LZ4, TEMP), print_output=True)): + if i > 0: + self.nvinfo_list.append(NVerboseFileInfo(line)) + + def test_frames(self): + all_concat_frames = 0 + all_concat_index = None + for i, nvinfo in enumerate(self.nvinfo_list): + if "concat-all" in nvinfo.filename: + all_concat_index = i + elif "2f--content-size" in nvinfo.filename: + self.assertEqual("2", nvinfo.frames, nvinfo.line) + all_concat_frames += 2 + else: + self.assertEqual("1", nvinfo.frames, nvinfo.line) + all_concat_frames += 1 + self.assertNotEqual(None, all_concat_index, "Couldn't find concat-all file index.") + self.assertEqual(self.nvinfo_list[all_concat_index].frames, str(all_concat_frames), self.nvinfo_list[all_concat_index].line) + + def test_frame_types(self): + for nvinfo in self.nvinfo_list: + if "-lz4f-" in nvinfo.filename: + self.assertEqual(nvinfo.type, "LZ4Frame", nvinfo.line) + elif "-legc-" in nvinfo.filename: + self.assertEqual(nvinfo.type, "LegacyFrame", nvinfo.line) + elif "-skip-" in nvinfo.filename: + self.assertEqual(nvinfo.type, "SkippableFrame", nvinfo.line) + + def test_block(self): + for nvinfo in self.nvinfo_list: + # if "-leg" in nvinfo.filename or "-skip" in nvinfo.filename: + # self.assertEqual(nvinfo.block, "-", nvinfo.line) + if "--BD" in nvinfo.filename: + self.assertRegex(nvinfo.block, "^B[0-9]+D$", nvinfo.line) + elif "--BI" in nvinfo.filename: + self.assertRegex(nvinfo.block, "^B[0-9]+I$", nvinfo.line) + + def test_compressed_size(self): + for nvinfo in self.nvinfo_list: + self.assertEqual(nvinfo.compressed, to_human(nvinfo.exp_comp_size), nvinfo.line) + + def test_ratio(self): + for nvinfo in self.nvinfo_list: + if "--content-size" in nvinfo.filename: + self.assertEqual(nvinfo.ratio, "{:.2f}%".format(float(nvinfo.exp_comp_size) / float(nvinfo.exp_unc_size) * 100), nvinfo.line) + + def test_uncompressed_size(self): + for nvinfo in self.nvinfo_list: + if "--content-size" in nvinfo.filename: + self.assertEqual(nvinfo.uncompressed, to_human(nvinfo.exp_unc_size), nvinfo.line) + + +class VerboseFileInfo(object): + def __init__(self, lines): + # Parse lines + self.frame_list = [] + self.file_frame_map = [] + for i, line in enumerate(lines): + if i == 0: + self.filename = line + continue + elif i == 1: + # Skip header + continue + frame_info = dict(zip(["frame", "type", "block", "checksum", "compressed", "uncompressed", "ratio"], line.split())) + frame_info["line"] = line + self.frame_list.append(frame_info) + + +class TestVerbose(unittest.TestCase): + @classmethod + def setUpClass(self): + # Even do we're listing 2 files to test multiline working as expected. + # we're only really interested in testing the output of the concat-all file. + self.vinfo_list = [] + start = end = 0 + output = execute("{} --list -m -v {}/test_list_concat-all.lz4 {}/test_list_*M-lz4f-2f--content-size.lz4".format(LZ4, TEMP, TEMP), print_output=True) + for i, line in enumerate(output): + if line.startswith("test_list"): + if start != 0 and end != 0: + self.vinfo_list.append(VerboseFileInfo(output[start:end])) + start = i + if not line: + end = i + self.vinfo_list.append(VerboseFileInfo(output[start:end])) + # Populate file_frame_map as a reference of the expected info + concat_file_list = glob.glob("/tmp/test_list_[!concat]*.lz4") + # One of the files has 2 frames so duplicate it in this list to map each frame 1 to a single file + for i, filename in enumerate(concat_file_list): + if "2f--content-size" in filename: + concat_file_list.insert(i, filename) + break + self.cvinfo = self.vinfo_list[0] + self.cvinfo.file_frame_map = concat_file_list + self.cvinfo.compressed_size = os.path.getsize("{}/test_list_concat-all.lz4".format(TEMP)) + + def test_filename(self): + for i, vinfo in enumerate(self.vinfo_list): + self.assertRegex(vinfo.filename, "^test_list_.*({}/{})".format(i + 1, len(self.vinfo_list))) + + def test_frame_number(self): + for vinfo in self.vinfo_list: + for i, frame_info in enumerate(vinfo.frame_list): + self.assertEqual(frame_info["frame"], str(i + 1), frame_info["line"]) + + def test_frame_type(self): + for i, frame_info in enumerate(self.cvinfo.frame_list): + if "-lz4f-" in self.cvinfo.file_frame_map[i]: + self.assertEqual(self.cvinfo.frame_list[i]["type"], "LZ4Frame", self.cvinfo.frame_list[i]["line"]) + elif "-legc-" in self.cvinfo.file_frame_map[i]: + self.assertEqual(self.cvinfo.frame_list[i]["type"], "LegacyFrame", self.cvinfo.frame_list[i]["line"]) + elif "-skip-" in self.cvinfo.file_frame_map[i]: + self.assertEqual(self.cvinfo.frame_list[i]["type"], "SkippableFrame", self.cvinfo.frame_list[i]["line"]) + + def test_block(self): + for i, frame_info in enumerate(self.cvinfo.frame_list): + if "--BD" in self.cvinfo.file_frame_map[i]: + self.assertRegex(self.cvinfo.frame_list[i]["block"], "^B[0-9]+D$", self.cvinfo.frame_list[i]["line"]) + elif "--BI" in self.cvinfo.file_frame_map[i]: + self.assertEqual(self.cvinfo.frame_list[i]["block"], "^B[0-9]+I$", self.cvinfo.frame_list[i]["line"]) + + def test_checksum(self): + for i, frame_info in enumerate(self.cvinfo.frame_list): + if "-lz4f-" in self.cvinfo.file_frame_map[i] and "--no-frame-crc" not in self.cvinfo.file_frame_map[i]: + self.assertEqual(self.cvinfo.frame_list[i]["checksum"], "XXH32", self.cvinfo.frame_list[i]["line"]) + + def test_compressed(self): + total = 0 + for i, frame_info in enumerate(self.cvinfo.frame_list): + if "-2f-" not in self.cvinfo.file_frame_map[i]: + expected_size = os.path.getsize(self.cvinfo.file_frame_map[i]) + self.assertEqual(self.cvinfo.frame_list[i]["compressed"], str(expected_size), self.cvinfo.frame_list[i]["line"]) + total += int(self.cvinfo.frame_list[i]["compressed"]) + self.assertEqual(total, self.cvinfo.compressed_size, "Expected total sum ({}) to match {} filesize".format(total, self.cvinfo.filename)) + + def test_uncompressed(self): + for i, frame_info in enumerate(self.cvinfo.frame_list): + ffm = self.cvinfo.file_frame_map[i] + if "-2f-" not in ffm and "--content-size" in ffm: + expected_size_unc = int(ffm[ffm.rindex("_") + 1:ffm.index("M")]) * 1048576 + self.assertEqual(self.cvinfo.frame_list[i]["uncompressed"], str(expected_size_unc), self.cvinfo.frame_list[i]["line"]) + + def test_ratio(self): + for i, frame_info in enumerate(self.cvinfo.frame_list): + if "--content-size" in self.cvinfo.file_frame_map[i]: + self.assertEqual(self.cvinfo.frame_list[i]['ratio'], + "{:.2f}%".format(float(self.cvinfo.frame_list[i]['compressed']) / float(self.cvinfo.frame_list[i]['uncompressed']) * 100), + self.cvinfo.frame_list[i]["line"]) + + +def to_human(size): + for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y']: + if size < 1024.0: + break + size /= 1024.0 + return "{:.2f}{}".format(size, unit) + + +def log(text): + print(time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text) + + +def errout(text, err=1): + log(text) + exit(err) + + +def execute(command, print_command=True, print_output=False, print_error=True, param_shell=True): + if os.environ.get('QEMU_SYS'): + command = "{} {}".format(os.environ['QEMU_SYS'], command) + if print_command: + log("> " + command) + popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=param_shell) + stdout_lines, stderr_lines = popen.communicate() + stderr_lines = stderr_lines.decode("utf-8") + stdout_lines = stdout_lines.decode("utf-8") + if print_output: + if stdout_lines: + print(stdout_lines) + if stderr_lines: + print(stderr_lines) + if popen.returncode is not None and popen.returncode != 0: + if stderr_lines and not print_output and print_error: + print(stderr_lines) + errout("Failed to run: {}\n".format(command, stdout_lines + stderr_lines)) + return (stdout_lines + stderr_lines).splitlines() + + +def cleanup(silent=False): + for f in glob.glob("{}/test_list*".format(TEMP)): + if not silent: + log("Deleting {}".format(f)) + os.unlink(f) + + +def datagen(file_name, size): + non_sparse_size = size // 2 + sparse_size = size - non_sparse_size + with open(file_name, "wb") as f: + f.seek(sparse_size) + f.write(os.urandom(non_sparse_size)) + + +def generate_files(): + # file format ~ test_list<frametype>-<no_frames>f<create-args>.lz4 ~ + # Generate LZ4Frames + for i in SIZES: + filename = "{}/test_list_{}M".format(TEMP, i) + log("Generating {}".format(filename)) + datagen(filename, i * MIB) + for j in ["--content-size", "-BI", "-BD", "-BX", "--no-frame-crc"]: + lz4file = "{}-lz4f-1f{}.lz4".format(filename, j) + execute("{} {} {} {}".format(LZ4, j, filename, lz4file)) + # Generate skippable frames + lz4file = "{}-skip-1f.lz4".format(filename) + skipsize = i * 1024 + skipbytes = bytes([80, 42, 77, 24]) + skipsize.to_bytes(4, byteorder='little', signed=False) + with open(lz4file, 'wb') as f: + f.write(skipbytes) + f.write(os.urandom(skipsize)) + # Generate legacy frames + lz4file = "{}-legc-1f.lz4".format(filename) + execute("{} -l {} {}".format(LZ4, filename, lz4file)) + + # Concatenate --content-size files + file_list = glob.glob("{}/test_list_*-lz4f-1f--content-size.lz4".format(TEMP)) + with open("{}/test_list_{}M-lz4f-2f--content-size.lz4".format(TEMP, sum(SIZES)), 'ab') as outfile: + for fname in file_list: + with open(fname, 'rb') as infile: + outfile.write(infile.read()) + + # Concatenate all files + file_list = glob.glob("{}/test_list_*.lz4".format(TEMP)) + with open("{}/test_list_concat-all.lz4".format(TEMP), 'ab') as outfile: + for fname in file_list: + with open(fname, 'rb') as infile: + outfile.write(infile.read()) + + +if __name__ == '__main__': + cleanup() + generate_files() + unittest.main(verbosity=2, exit=False) + cleanup(silent=True) |