From af58d2369c8d4614ccd69452617b283e6562768e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 4 Dec 2019 08:49:06 -0800 Subject: added test for LZ4_decompress_safe_withPrefix64k() --- tests/fullbench.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/fullbench.c b/tests/fullbench.c index 77f475b..a0c42b4 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -24,8 +24,8 @@ */ -// S_ISREG & gettimeofday() are not supported by MSVC #if defined(_MSC_VER) || defined(_WIN32) + /* S_ISREG & gettimeofday() are not supported by MSVC */ # define BMK_LEGACY_TIMER 1 #endif @@ -134,7 +134,7 @@ static clock_t BMK_GetClockSpan( clock_t clockStart ) static size_t BMK_findMaxMem(U64 requiredMem) { size_t step = 64 MB; - BYTE* testmem=NULL; + BYTE* testmem = NULL; requiredMem = (((requiredMem >> 26) + 1) << 26); requiredMem += 2*step; @@ -292,9 +292,14 @@ static int local_LZ4_decompress_fast_usingExtDict(const char* in, char* out, int return outSize; } +static int local_LZ4_decompress_safe_withPrefix64k(const char* in, char* out, int inSize, int outSize) +{ + LZ4_decompress_safe_withPrefix64k(in, out, inSize, outSize); + return outSize; +} + static int local_LZ4_decompress_safe_usingDict(const char* in, char* out, int inSize, int outSize) { - (void)inSize; LZ4_decompress_safe_usingDict(in, out, inSize, outSize, out - 65536, 65536); return outSize; } @@ -608,6 +613,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) case 2: decompressionFunction = local_LZ4_decompress_fast_usingDict_prefix; dName = "LZ4_decompress_fast_usingDict(prefix)"; break; case 3: decompressionFunction = local_LZ4_decompress_fast_usingExtDict; dName = "LZ4_decompress_fast_using(Ext)Dict"; break; case 4: decompressionFunction = LZ4_decompress_safe; dName = "LZ4_decompress_safe"; break; + case 5: decompressionFunction = local_LZ4_decompress_safe_withPrefix64k; dName = "LZ4_decompress_safe_withPrefix64k"; break; case 6: decompressionFunction = local_LZ4_decompress_safe_usingDict; dName = "LZ4_decompress_safe_usingDict"; break; case 7: decompressionFunction = local_LZ4_decompress_safe_partial; dName = "LZ4_decompress_safe_partial"; checkResult = 0; break; #ifndef LZ4_DLL_IMPORT -- cgit v0.12 From 38b77ece91aba6c72bc00590f31762f45da6e8fa Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 4 Dec 2019 08:57:12 -0800 Subject: fullbench: added LZ4F_decompress_noHint() --- tests/fullbench.c | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/fullbench.c b/tests/fullbench.c index a0c42b4..4ec30ce 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -386,6 +386,39 @@ static int local_LZ4F_decompress_followHint(const char* src, char* dst, int srcS } +/* always provide input by block of 64 KB */ +static int local_LZ4F_decompress_noHint(const char* src, char* dst, int srcSize, int dstSize) +{ + size_t totalInSize = (size_t)srcSize; + size_t maxOutSize = (size_t)dstSize; + + size_t inPos = 0; + size_t inSize = 64 KB; + size_t outPos = 0; + size_t outRemaining = maxOutSize - outPos; + + for (;;) { + size_t const sizeHint = LZ4F_decompress(g_dCtx, dst+outPos, &outRemaining, src+inPos, &inSize, NULL); + assert(!LZ4F_isError(sizeHint)); + + inPos += inSize; + inSize = (inPos + 64 KB <= totalInSize) ? 64 KB : totalInSize - inPos; + + outPos += outRemaining; + outRemaining = maxOutSize - outPos; + + if (!sizeHint) break; + } + + /* frame completed */ + if (inPos != totalInSize) { + DISPLAY("Error decompressing frame : must read (%u) full frame (%u) \n", + (unsigned)inPos, (unsigned)totalInSize); + exit(10); + } + return (int)outPos; + +} #define NB_COMPRESSION_ALGORITHMS 100 #define NB_DECOMPRESSION_ALGORITHMS 100 @@ -621,8 +654,10 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) #endif case 10: case 11: + case 12: if (dAlgNb == 10) { decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress"; } /* can be skipped */ if (dAlgNb == 11) { decompressionFunction = local_LZ4F_decompress_followHint; dName = "LZ4F_decompress_followHint"; } /* can be skipped */ + if (dAlgNb == 12) { decompressionFunction = local_LZ4F_decompress_noHint; dName = "LZ4F_decompress_noHint"; } /* can be skipped */ /* prepare compressed data using frame format */ { size_t const fcsize = LZ4F_compressFrame(compressed_buff, (size_t)compressedBuffSize, orig_buff, benchedSize, NULL); assert(!LZ4F_isError(fcsize)); -- cgit v0.12 From 2964b8a6f6d93e86af52f2d23ec18f02421ebd74 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 8 Nov 2020 13:21:58 -0800 Subject: fix #874 coverity reported a warning regarding a memcpy() overwrite. This is a false positive (the memory area is large enough), but it's true that it's not trivial to determine (encompassing struct), and it's proper anyway to only memcpy() just the right amount of data. --- lib/lz4.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/lz4.c b/lib/lz4.c index 427673e..aca97ed 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -1606,7 +1606,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, * cost to copy the dictionary's tables into the active context, * so that the compression loop is only looking into one table. */ - LZ4_memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t)); + LZ4_memcpy(streamPtr, streamPtr->dictCtx, sizeof(*streamPtr)); result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); } else { result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration); -- cgit v0.12 From 2a2b10f19242aedc8598b74bcb4614d03485c3c0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 8 Nov 2020 18:08:43 -0800 Subject: fixed remaining ubsan warnings --- lib/lz4.c | 2 +- lib/lz4hc.c | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index 427673e..b066c55 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -1662,7 +1662,7 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; } if (safeBuffer == NULL) assert(dictSize == 0); - if (safeBuffer != NULL) + if (dictSize > 0) memmove(safeBuffer, previousDictEnd - dictSize, dictSize); dict->dictionary = (const BYTE*)safeBuffer; diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 8875f1a..286ff68 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -1164,13 +1164,16 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS if (dictSize > 64 KB) dictSize = 64 KB; if (dictSize < 4) dictSize = 0; if (dictSize > prefixSize) dictSize = prefixSize; - memmove(safeBuffer, streamPtr->end - dictSize, dictSize); + if (safeBuffer == NULL) assert(dictSize == 0); + if (dictSize > 0) + memmove(safeBuffer, streamPtr->end - dictSize, dictSize); { U32 const endIndex = (U32)(streamPtr->end - streamPtr->base); streamPtr->end = (const BYTE*)safeBuffer + dictSize; streamPtr->base = streamPtr->end - endIndex; streamPtr->dictLimit = endIndex - (U32)dictSize; streamPtr->lowLimit = endIndex - (U32)dictSize; - if (streamPtr->nextToUpdate < streamPtr->dictLimit) streamPtr->nextToUpdate = streamPtr->dictLimit; + if (streamPtr->nextToUpdate < streamPtr->dictLimit) + streamPtr->nextToUpdate = streamPtr->dictLimit; } return dictSize; } -- cgit v0.12 From c76564b944d1fe6e4ffc6dc3d002becfc3c74b5d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 8 Nov 2020 19:48:47 -0800 Subject: attempt at silencing cppcheck --- tests/fuzzer.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 99361dd..ba36621 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -341,7 +341,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c exit(1); \ } -# define FUZ_CHECKTEST(cond, ...) { if (cond) { EXIT_MSG(__VA_ARGS__) } } +# define FUZ_CHECKTEST(cond, ...) if (cond) { EXIT_MSG(__VA_ARGS__) } # define FUZ_DISPLAYTEST(...) { \ testNb++; \ @@ -1135,7 +1135,7 @@ static void FUZ_unitTests(int compressionLevel) shct* const shc = (shct*)malloc(sizeof(*shc)); assert(shc != NULL); memset(shc, 0, sizeof(*shc)); - DISPLAYLEVEL(3, "state1(%p) state2(%p) state3(%p) LZ4_stream_t size(0x%x): ", + DISPLAYLEVEL(4, "state1(%p) state2(%p) state3(%p) LZ4_stream_t size(0x%x): ", &(shc->state1), &(shc->state2), &(shc->state3), (unsigned)sizeof(LZ4_stream_t)); FUZ_CHECKTEST( LZ4_initStream(&(shc->state1), sizeof(shc->state1)) == NULL, "state1 (%p) failed init", &(shc->state1) ); FUZ_CHECKTEST( LZ4_initStream(&(shc->state2), sizeof(shc->state2)) == NULL, "state2 (%p) failed init", &(shc->state2) ); @@ -1156,16 +1156,16 @@ static void FUZ_unitTests(int compressionLevel) { LZ4_stream_t streamingState; /* simple compression test */ - { U64 const crcOrig = XXH64(testInput, testCompressedSize, 0); - LZ4_initStream(&streamingState, sizeof(streamingState)); - { int const cs = LZ4_compress_fast_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1, 1); - FUZ_CHECKTEST(cs==0, "LZ4_compress_fast_continue() compression failed!"); - { int const r = LZ4_decompress_safe(testCompressed, testVerify, cs, testCompressedSize); - FUZ_CHECKTEST(r!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); - } } - { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); + LZ4_initStream(&streamingState, sizeof(streamingState)); + { int const cs = LZ4_compress_fast_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1, 1); + FUZ_CHECKTEST(cs==0, "LZ4_compress_fast_continue() compression failed!"); + { int const r = LZ4_decompress_safe(testCompressed, testVerify, cs, testCompressedSize); + FUZ_CHECKTEST(r!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); } } + { U64 const crcOrig = XXH64(testInput, testCompressedSize, 0); + U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); + } /* early saveDict */ DISPLAYLEVEL(3, "saveDict (right after init) : "); @@ -1245,7 +1245,7 @@ static void FUZ_unitTests(int compressionLevel) shct* const shc = (shct*)malloc(sizeof(*shc)); assert(shc != NULL); memset(shc, 0, sizeof(*shc)); - DISPLAYLEVEL(3, "hc1(%p) hc2(%p) hc3(%p) size(0x%x): ", + DISPLAYLEVEL(4, "hc1(%p) hc2(%p) hc3(%p) size(0x%x): ", &(shc->hc1), &(shc->hc2), &(shc->hc3), (unsigned)sizeof(LZ4_streamHC_t)); FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc1), sizeof(shc->hc1)) == NULL, "hc1 (%p) failed init", &(shc->hc1) ); FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc2), sizeof(shc->hc2)) == NULL, "hc2 (%p) failed init", &(shc->hc2) ); -- cgit v0.12