diff options
author | Yann Collet <Cyan4973@users.noreply.github.com> | 2020-09-28 21:04:56 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-09-28 21:04:56 (GMT) |
commit | 78f4fdbb89e71325cedc751e1f7c2403bb43c3f4 (patch) | |
tree | 342568871cc4c17b6c1a914e89def841c1ea9eaa /tests/fuzzer.c | |
parent | 8a362a8ac890b820cc16071bf190740445dfc7a2 (diff) | |
parent | ab89dda91d2c878c1f0638dabacf97b5a68ec361 (diff) | |
download | lz4-78f4fdbb89e71325cedc751e1f7c2403bb43c3f4.zip lz4-78f4fdbb89e71325cedc751e1f7c2403bb43c3f4.tar.gz lz4-78f4fdbb89e71325cedc751e1f7c2403bb43c3f4.tar.bz2 |
Merge pull request #923 from lz4/fix784
fix efficiency of LZ4_compress_HC_destSize()
Diffstat (limited to 'tests/fuzzer.c')
-rw-r--r-- | tests/fuzzer.c | 116 |
1 files changed, 101 insertions, 15 deletions
diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 4dbf6a7..6c2ccb7 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -446,10 +446,10 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_DISPLAYTEST(); decodedBuffer[srcSize] = canary; { int const dSize = LZ4_decompress_safe(compressedBuffer, decodedBuffer, cSize, srcSize); - FUZ_CHECKTEST(dSize<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compressHC_destSize"); - FUZ_CHECKTEST(dSize!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data"); + FUZ_CHECKTEST(dSize<0, "LZ4_decompress_safe failed (%i) on data compressed by LZ4_compressHC_destSize", dSize); + FUZ_CHECKTEST(dSize!=srcSize, "LZ4_decompress_safe failed : decompressed %i bytes, was supposed to decompress %i bytes", dSize, srcSize); } - FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe() overwrite dst buffer !"); + FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe overwrite dst buffer !"); { U32 const crcDec = XXH32(decodedBuffer, (size_t)srcSize, 0); FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); } } @@ -931,7 +931,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c LZ4_resetStreamHC_fast (LZ4_streamHC, compressionLevel); LZ4_attach_HC_dictionary(LZ4_streamHC, LZ4dictHC); ret = LZ4_compress_HC_continue(LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize); - FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx and fast reset size is different (%i != %i)", ret, blockContinueCompressedSize); + FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx and fast reset size is different (%i != %i)", + ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue using ExtDictCtx and fast reset should work : enough size available within output buffer"); FUZ_CHECKTEST(LZ4_streamHC->internal_donotuse.dirty, "Context should be clean"); @@ -957,7 +958,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c LZ4_loadDictHC(LZ4dictHC, dict, dictSize); LZ4_setCompressionLevel(LZ4dictHC, compressionLevel); blockContinueCompressedSize = LZ4_compress_HC_continue_destSize(LZ4dictHC, block, compressedBuffer, &consumedSize, availableSpace); - DISPLAYLEVEL(5, " LZ4_compress_HC_continue_destSize : compressed %6i/%6i into %6i/%6i at cLevel=%i\n", consumedSize, blockSize, blockContinueCompressedSize, availableSpace, compressionLevel); + DISPLAYLEVEL(5, " LZ4_compress_HC_continue_destSize : compressed %6i/%6i into %6i/%6i at cLevel=%i \n", + consumedSize, blockSize, blockContinueCompressedSize, availableSpace, compressionLevel); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue_destSize failed"); FUZ_CHECKTEST(blockContinueCompressedSize > availableSpace, "LZ4_compress_HC_continue_destSize write overflow"); FUZ_CHECKTEST(consumedSize > blockSize, "LZ4_compress_HC_continue_destSize read overflow"); @@ -965,7 +967,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_DISPLAYTEST(); decodedBuffer[consumedSize] = 0; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, consumedSize, dict, dictSize); - FUZ_CHECKTEST(ret!=consumedSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); + FUZ_CHECKTEST(ret != consumedSize, "LZ4_decompress_safe_usingDict regenerated %i bytes (%i expected)", ret, consumedSize); FUZ_CHECKTEST(decodedBuffer[consumedSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size") { U32 const crcSrc = XXH32(block, (size_t)consumedSize, 0); U32 const crcDst = XXH32(decodedBuffer, (size_t)consumedSize, 0); @@ -1200,12 +1202,11 @@ static void FUZ_unitTests(int compressionLevel) /* LZ4 HC streaming tests */ { LZ4_streamHC_t sHC; /* statically allocated */ - U64 crcOrig; int result; LZ4_initStreamHC(&sHC, sizeof(sHC)); /* Allocation test */ - DISPLAYLEVEL(3, " Basic HC allocation : "); + DISPLAYLEVEL(3, "Basic HC allocation : "); { LZ4_streamHC_t* const sp = LZ4_createStreamHC(); FUZ_CHECKTEST(sp==NULL, "LZ4_createStreamHC() allocation failed"); LZ4_freeStreamHC(sp); @@ -1213,7 +1214,7 @@ static void FUZ_unitTests(int compressionLevel) DISPLAYLEVEL(3, " OK \n"); /* simple HC compression test */ - DISPLAYLEVEL(3, " Simple HC round-trip : "); + DISPLAYLEVEL(3, "Simple HC round-trip : "); { U64 const crc64 = XXH64(testInput, testCompressedSize, 0); LZ4_setCompressionLevel(&sHC, compressionLevel); result = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1); @@ -1228,7 +1229,7 @@ static void FUZ_unitTests(int compressionLevel) DISPLAYLEVEL(3, " OK \n"); /* long sequence test */ - DISPLAYLEVEL(3, " Long sequence HC test : "); + DISPLAYLEVEL(3, "Long sequence HC_destSize test : "); { size_t const blockSize = 1 MB; size_t const targetSize = 4116; /* size carefully selected to trigger an overflow */ void* const block = malloc(blockSize); @@ -1246,8 +1247,13 @@ static void FUZ_unitTests(int compressionLevel) assert(targetSize < INT_MAX); result = LZ4_compress_HC_destSize(&sHC, (const char*)block, (char*)dstBlock, &srcSize, (int)targetSize, 3); DISPLAYLEVEL(4, "cSize=%i; readSize=%i; ", result, srcSize); - FUZ_CHECKTEST(result!=4116, "LZ4_compress_HC_destSize() : compression must fill dstBuffer completely, but no more !"); - FUZ_CHECKTEST(((char*)dstBlock)[targetSize] != sentinel, "LZ4_compress_HC_destSize()") + FUZ_CHECKTEST(result != 4116, "LZ4_compress_HC_destSize() : " + "compression (%i->%i) must fill dstBuffer (%i) exactly", + srcSize, result, (int)targetSize); + FUZ_CHECKTEST(((char*)dstBlock)[targetSize] != sentinel, + "LZ4_compress_HC_destSize() overwrites dst buffer"); + FUZ_CHECKTEST(srcSize < 1045000, "LZ4_compress_HC_destSize() doesn't compress enough" + " (%i -> %i , expected > %i)", srcSize, result, 1045000); LZ4_resetStreamHC_fast(&sHC, 3); /* make sure the context is clean after the test */ free(block); @@ -1256,7 +1262,7 @@ static void FUZ_unitTests(int compressionLevel) DISPLAYLEVEL(3, " OK \n"); /* simple dictionary HC compression test */ - DISPLAYLEVEL(3, " HC dictionary compression test : "); + DISPLAYLEVEL(3, "HC dictionary compression test : "); { U64 const crc64 = XXH64(testInput + 64 KB, testCompressedSize, 0); LZ4_resetStreamHC_fast(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, 64 KB); @@ -1325,6 +1331,7 @@ static void FUZ_unitTests(int compressionLevel) XXH64_reset(&crcNewState, 0); while (segStart + segSize < testInputSize) { + XXH64_hash_t crcOrig; XXH64_update(&crcOrigState, testInput + segStart, segSize); crcOrig = XXH64_digest(&crcOrigState); assert(segSize <= INT_MAX); @@ -1371,6 +1378,7 @@ static void FUZ_unitTests(int compressionLevel) while (iNext + messageSize < testCompressedSize) { int compressedSize; + XXH64_hash_t crcOrig; XXH64_update(&xxhOrig, testInput + iNext, messageSize); crcOrig = XXH64_digest(&xxhOrig); @@ -1385,7 +1393,7 @@ static void FUZ_unitTests(int compressionLevel) FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe_continue() test failed"); XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize); - { U64 const crcNew = XXH64_digest(&xxhNewSafe); + { XXH64_hash_t const crcNew = XXH64_digest(&xxhNewSafe); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); } assert(messageSize < INT_MAX); @@ -1393,7 +1401,7 @@ static void FUZ_unitTests(int compressionLevel) FUZ_CHECKTEST(result!=compressedSize, "ringBuffer : LZ4_decompress_fast_continue() test failed"); XXH64_update(&xxhNewFast, testVerify + dNext, messageSize); - { U64 const crcNew = XXH64_digest(&xxhNewFast); + { XXH64_hash_t const crcNew = XXH64_digest(&xxhNewFast); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption"); } /* prepare next message */ @@ -1414,6 +1422,7 @@ static void FUZ_unitTests(int compressionLevel) */ { XXH64_state_t xxhOrig; XXH64_state_t xxhNewSafe, xxhNewFast; + XXH64_hash_t crcOrig; LZ4_streamDecode_t decodeStateSafe, decodeStateFast; const int maxMessageSizeLog = 12; const int maxMessageSize = 1 << maxMessageSizeLog; @@ -1510,8 +1519,85 @@ static void FUZ_unitTests(int compressionLevel) iNext = (FUZ_rand(&randState) & 65535); if (dNext + maxMessageSize > dBufferSize) dNext = 0; } + } /* Ring buffer test : Non synchronized decoder */ + } + + DISPLAYLEVEL(3, "LZ4_compress_HC_destSize : "); + /* encode congenerical sequence test for HC compressors */ + { LZ4_streamHC_t sHC; /* statically allocated */ + int const src_buf_size = 3 MB; + int const dst_buf_size = 6 KB; + int const payload = 0; + int const dst_step = 43; + int const dst_min_len = 33 + (FUZ_rand(&randState) % dst_step); + int const dst_max_len = 5000; + int slen, dlen; + char* sbuf1 = (char*)malloc(src_buf_size + 1); + char* sbuf2 = (char*)malloc(src_buf_size + 1); + char* dbuf1 = (char*)malloc(dst_buf_size + 1); + char* dbuf2 = (char*)malloc(dst_buf_size + 1); + + assert(dst_buf_size > dst_max_len); + if (!sbuf1 || !sbuf2 || !dbuf1 || !dbuf2) { + EXIT_MSG("not enough memory for FUZ_unitTests (destSize)"); + } + for (dlen = dst_min_len; dlen <= dst_max_len; dlen += dst_step) { + int src_len = (dlen - 10)*255 + 24; + if (src_len + 10 >= src_buf_size) break; /* END of check */ + for (slen = src_len - 3; slen <= src_len + 3; slen++) { + int srcsz1, srcsz2; + int dsz1, dsz2; + int res1, res2; + char const endchk = (char)0x88; + DISPLAYLEVEL(5, "slen = %i, ", slen); + + srcsz1 = slen; + memset(sbuf1, payload, slen); + memset(dbuf1, 0, dlen); + dbuf1[dlen] = endchk; + dsz1 = LZ4_compress_destSize(sbuf1, dbuf1, &srcsz1, dlen); + DISPLAYLEVEL(5, "LZ4_compress_destSize: %i bytes compressed into %i bytes, ", srcsz1, dsz1); + DISPLAYLEVEL(5, "last token : 0x%0X, ", dbuf1[dsz1 - 6]); + DISPLAYLEVEL(5, "last ML extra lenbyte : 0x%0X, \n", dbuf1[dsz1 - 7]); + FUZ_CHECKTEST(dbuf1[dlen] != endchk, "LZ4_compress_destSize() overwrite dst buffer !"); + FUZ_CHECKTEST(dsz1 <= 0, "LZ4_compress_destSize() compression failed"); + FUZ_CHECKTEST(dsz1 > dlen, "LZ4_compress_destSize() result larger than dst buffer !"); + FUZ_CHECKTEST(srcsz1 > slen, "LZ4_compress_destSize() read more than src buffer !"); + + res1 = LZ4_decompress_safe(dbuf1, sbuf1, dsz1, src_buf_size); + FUZ_CHECKTEST(res1 != srcsz1, "LZ4_compress_destSize() decompression failed!"); + + srcsz2 = slen; + memset(sbuf2, payload, slen); + memset(dbuf2, 0, dlen); + dbuf2[dlen] = endchk; + LZ4_resetStreamHC(&sHC, compressionLevel); + dsz2 = LZ4_compress_HC_destSize(&sHC, sbuf2, dbuf2, &srcsz2, dlen, compressionLevel); + DISPLAYLEVEL(5, "LZ4_compress_HC_destSize: %i bytes compressed into %i bytes, ", srcsz2, dsz2); + DISPLAYLEVEL(5, "last token : 0x%0X, ", dbuf2[dsz2 - 6]); + DISPLAYLEVEL(5, "last ML extra lenbyte : 0x%0X, \n", dbuf2[dsz2 - 7]); + FUZ_CHECKTEST(dbuf2[dlen] != endchk, "LZ4_compress_HC_destSize() overwrite dst buffer !"); + FUZ_CHECKTEST(dsz2 <= 0, "LZ4_compress_HC_destSize() compression failed"); + FUZ_CHECKTEST(dsz2 > dlen, "LZ4_compress_HC_destSize() result larger than dst buffer !"); + FUZ_CHECKTEST(srcsz2 > slen, "LZ4_compress_HC_destSize() read more than src buffer !"); + FUZ_CHECKTEST(dsz2 != dsz1, "LZ4_compress_HC_destSize() return incorrect result !"); + FUZ_CHECKTEST(srcsz2 != srcsz1, "LZ4_compress_HC_destSize() return incorrect src buffer size " + ": srcsz2(%i) != srcsz1(%i)", srcsz2, srcsz1); + FUZ_CHECKTEST(memcmp(dbuf2, dbuf1, (size_t)dsz2), "LZ4_compress_HC_destSize() return incorrect data into dst buffer !"); + + res2 = LZ4_decompress_safe(dbuf2, sbuf1, dsz2, src_buf_size); + FUZ_CHECKTEST(res2 != srcsz1, "LZ4_compress_HC_destSize() decompression failed!"); + + FUZ_CHECKTEST(memcmp(sbuf1, sbuf2, (size_t)res2), "LZ4_compress_HC_destSize() decompression corruption!"); + } } + free(sbuf1); + free(sbuf2); + free(dbuf1); + free(dbuf2); } + DISPLAYLEVEL(3, " OK \n"); + /* clean up */ free(testInput); |