diff options
| -rw-r--r-- | doc/lz4_manual.html | 59 | ||||
| -rw-r--r-- | lib/lz4.c | 40 | ||||
| -rw-r--r-- | lib/lz4.h | 83 | ||||
| -rw-r--r-- | lib/lz4frame.c | 2 | ||||
| -rw-r--r-- | lib/lz4hc.h | 3 | ||||
| -rw-r--r-- | tests/fuzzer.c | 164 |
6 files changed, 269 insertions, 82 deletions
diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html index 3a9e0db..1480089 100644 --- a/doc/lz4_manual.html +++ b/doc/lz4_manual.html @@ -40,9 +40,9 @@ Blocks are different from Frames (doc/lz4_Frame_format.md). Frames bundle both blocks and metadata in a specified manner. - This are required for compressed data to be self-contained and portable. + Embedding metadata is required for compressed data to be self-contained and portable. Frame format is delivered through a companion API, declared in lz4frame.h. - Note that the `lz4` CLI can only manage frames. + The `lz4` CLI can only manage frames. <BR></pre> <a name="Chapter2"></a><h2>Version</h2><pre></pre> @@ -357,6 +357,61 @@ int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); </p></pre><BR> +<pre><b></b><p> + It's possible to have input and output sharing the same buffer, + for highly contrained memory environments. + In both cases, it requires input to lay at the end of the buffer, + and decompression to start at beginning of the buffer. + Buffer size must feature some margin, hence be larger than final size. + + |<------------------------buffer--------------------------------->| + |<-----------compressed data--------->| + |<-----------decompressed size------------------>| + |<----margin---->| + + This technique is more useful for decompression, + since decompressed size is typically larger, + and margin is short. + + In-place decompression will work inside any buffer + which size is >= LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize). + This presumes that decompressedSize > compressedSize. + Otherwise, it means compression actually expanded data, + and it would be more efficient to store such data with a flag indicating it's not compressed. + This can happen when data is not compressible (already compressed, or encrypted). + + For in-place compression, margin is larger, as it must be able to cope with both + history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX, + and data expansion, which can happen when input is not compressible. + As a consequence, buffer size requirements are much higher, + and memory savings offered by in-place compression are more limited. + + There are ways to limit this cost for compression : + - Reduce history size, by modifying LZ4_DISTANCE_MAX. + Note that it is a compile-time constant, so all compressions will apply this limit. + Lower values will reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX, + so it's a reasonable trick when inputs are known to be small. + - Require the compressor to deliver a "maximum compressed size". + This is the `dstCapacity` parameter in `LZ4_compress*()`. + When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail, + in which case, the return code will be 0 (zero). + The caller must be ready for these cases to happen, + and typically design a backup scheme to send data uncompressed. + The combination of both techniques can significantly reduce + the amount of margin required for in-place compression. + + In-place compression can work in any buffer + which size is >= (maxCompressedSize) + with maxCompressedSize == LZ4_COMPRESSBOUND(srcSize) for guaranteed compression success. + LZ4_COMPRESS_INPLACE_BUFFER_SIZE() depends on both maxCompressedSize and LZ4_DISTANCE_MAX, + so it's possible to reduce memory requirements by playing with them. + +</p></pre><BR> + +<pre><b>#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize) ((decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize)) </b>/**< note: presumes that compressedSize < decompressedSize. note2: margin is overestimated a bit, since it could use compressedSize instead */<b> +</b></pre><BR> +<pre><b>#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize) ((maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN) </b>/**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */<b> +</b></pre><BR> <a name="Chapter9"></a><h2>PRIVATE DEFINITIONS</h2><pre> Do not use these definitions directly. They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. @@ -106,6 +106,7 @@ #define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */ #endif +#define LZ4_STATIC_LINKING_ONLY /* LZ4_DISTANCE_MAX */ #include "lz4.h" /* see also "memory routines" below */ @@ -119,6 +120,7 @@ # pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ #endif /* _MSC_VER */ +#define LZ4_FORCE_INLINE static #ifndef LZ4_FORCE_INLINE # ifdef _MSC_VER /* Visual Studio */ # define LZ4_FORCE_INLINE static __forceinline @@ -412,10 +414,6 @@ static const int LZ4_minLength = (MFLIMIT+1); #define MB *(1 <<20) #define GB *(1U<<30) -#ifndef LZ4_DISTANCE_MAX /* can be user - defined at compile time */ -# define LZ4_DISTANCE_MAX 65535 -#endif - #if (LZ4_DISTANCE_MAX > 65535) /* max supported by LZ4 format */ # error "LZ4_DISTANCE_MAX is too big : must be <= 65535" #endif @@ -465,7 +463,7 @@ static unsigned LZ4_NbCommonBytes (reg_t val) _BitScanForward64( &r, (U64)val ); return (int)(r>>3); # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctzll((U64)val) >> 3); + return (unsigned)__builtin_ctzll((U64)val) >> 3; # else static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, @@ -483,7 +481,7 @@ static unsigned LZ4_NbCommonBytes (reg_t val) _BitScanForward( &r, (U32)val ); return (int)(r>>3); # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctz((U32)val) >> 3); + return (unsigned)__builtin_ctz((U32)val) >> 3; # else static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, @@ -499,7 +497,7 @@ static unsigned LZ4_NbCommonBytes (reg_t val) _BitScanReverse64( &r, val ); return (unsigned)(r>>3); # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clzll((U64)val) >> 3); + return (unsigned)__builtin_clzll((U64)val) >> 3; # else static const U32 by32 = sizeof(val)*4; /* 32 on 64 bits (goal), 16 on 32 bits. Just to avoid some static analyzer complaining about shift by 32 on 32-bits target. @@ -516,7 +514,7 @@ static unsigned LZ4_NbCommonBytes (reg_t val) _BitScanReverse( &r, (unsigned long)val ); return (unsigned)(r>>3); # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clz((U32)val) >> 3); + return (unsigned)__builtin_clz((U32)val) >> 3; # else unsigned r; if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } @@ -710,18 +708,19 @@ static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType { const U16* const hashTable = (const U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ } -LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, - const void* tableBase, tableType_t tableType, - const BYTE* srcBase) +LZ4_FORCE_INLINE const BYTE* +LZ4_getPosition(const BYTE* p, + const void* tableBase, tableType_t tableType, + const BYTE* srcBase) { U32 const h = LZ4_hashPosition(p, tableType); return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); } -LZ4_FORCE_INLINE void LZ4_prepareTable( - LZ4_stream_t_internal* const cctx, - const int inputSize, - const tableType_t tableType) { +LZ4_FORCE_INLINE void +LZ4_prepareTable(LZ4_stream_t_internal* const cctx, + const int inputSize, + const tableType_t tableType) { /* If compression failed during the previous step, then the context * is marked as dirty, therefore, it has to be fully reset. */ @@ -736,9 +735,10 @@ LZ4_FORCE_INLINE void LZ4_prepareTable( * out if it's safe to leave as is or whether it needs to be reset. */ if (cctx->tableType != clearedTable) { + assert(inputSize >= 0); if (cctx->tableType != tableType - || (tableType == byU16 && cctx->currentOffset + inputSize >= 0xFFFFU) - || (tableType == byU32 && cctx->currentOffset > 1 GB) + || ((tableType == byU16) && cctx->currentOffset + (unsigned)inputSize >= 0xFFFFU) + || ((tableType == byU32) && cctx->currentOffset > 1 GB) || tableType == byPtr || inputSize >= 4 KB) { @@ -1708,10 +1708,9 @@ LZ4_decompress_generic( /* get matchlength */ length = token & ML_MASK; - if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ - if (length == ML_MASK) { variable_length_error error = ok; + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error); if (error != ok) { goto _output_error; } if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */ @@ -1735,6 +1734,7 @@ LZ4_decompress_generic( continue; } } } + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ /* match starting within external dictionary */ if ((dict==usingExtDict) && (match < lowPrefix)) { if (unlikely(op+length > oend-LASTLITERALS)) { @@ -1853,7 +1853,7 @@ LZ4_decompress_generic( if ((!endOnInput) && (cpy != oend)) { goto _output_error; } /* Error : block decoding must stop exactly there */ if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) { goto _output_error; } /* Error : input must be consumed */ } - memcpy(op, ip, length); + memmove(op, ip, length); /* supports overlapping memory regions, which only matters for in-place decompression scenarios */ ip += length; op += length; if (!partialDecoding || (cpy == oend)) { @@ -65,9 +65,9 @@ extern "C" { Blocks are different from Frames (doc/lz4_Frame_format.md). Frames bundle both blocks and metadata in a specified manner. - This are required for compressed data to be self-contained and portable. + Embedding metadata is required for compressed data to be self-contained and portable. Frame format is delivered through a companion API, declared in lz4frame.h. - Note that the `lz4` CLI can only manage frames. + The `lz4` CLI can only manage frames. */ /*^*************************************************************** @@ -388,6 +388,8 @@ LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecod */ LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize); +#endif /* LZ4_H_2983827168210 */ + /*^************************************* * !!!!!! STATIC LINKING ONLY !!!!!! @@ -413,14 +415,17 @@ LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int sr * define LZ4_PUBLISH_STATIC_FUNCTIONS when building the LZ4 library. ******************************************************************************/ +#ifdef LZ4_STATIC_LINKING_ONLY + +#ifndef LZ4_STATIC_3504398509 +#define LZ4_STATIC_3504398509 + #ifdef LZ4_PUBLISH_STATIC_FUNCTIONS #define LZ4LIB_STATIC_API LZ4LIB_API #else #define LZ4LIB_STATIC_API #endif -#ifdef LZ4_STATIC_LINKING_ONLY - /*! LZ4_compress_fast_extState_fastReset() : * A variant of LZ4_compress_fast_extState(). @@ -462,8 +467,75 @@ LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const c */ LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream); + +/*! In-place compression and decompression + * + * It's possible to have input and output sharing the same buffer, + * for highly contrained memory environments. + * In both cases, it requires input to lay at the end of the buffer, + * and decompression to start at beginning of the buffer. + * Buffer size must feature some margin, hence be larger than final size. + * + * |<------------------------buffer--------------------------------->| + * |<-----------compressed data--------->| + * |<-----------decompressed size------------------>| + * |<----margin---->| + * + * This technique is more useful for decompression, + * since decompressed size is typically larger, + * and margin is short. + * + * In-place decompression will work inside any buffer + * which size is >= LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize). + * This presumes that decompressedSize > compressedSize. + * Otherwise, it means compression actually expanded data, + * and it would be more efficient to store such data with a flag indicating it's not compressed. + * This can happen when data is not compressible (already compressed, or encrypted). + * + * For in-place compression, margin is larger, as it must be able to cope with both + * history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX, + * and data expansion, which can happen when input is not compressible. + * As a consequence, buffer size requirements are much higher, + * and memory savings offered by in-place compression are more limited. + * + * There are ways to limit this cost for compression : + * - Reduce history size, by modifying LZ4_DISTANCE_MAX. + * Note that it is a compile-time constant, so all compressions will apply this limit. + * Lower values will reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX, + * so it's a reasonable trick when inputs are known to be small. + * - Require the compressor to deliver a "maximum compressed size". + * This is the `dstCapacity` parameter in `LZ4_compress*()`. + * When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail, + * in which case, the return code will be 0 (zero). + * The caller must be ready for these cases to happen, + * and typically design a backup scheme to send data uncompressed. + * The combination of both techniques can significantly reduce + * the amount of margin required for in-place compression. + * + * In-place compression can work in any buffer + * which size is >= (maxCompressedSize) + * with maxCompressedSize == LZ4_COMPRESSBOUND(srcSize) for guaranteed compression success. + * LZ4_COMPRESS_INPLACE_BUFFER_SIZE() depends on both maxCompressedSize and LZ4_DISTANCE_MAX, + * so it's possible to reduce memory requirements by playing with them. + */ + +#define LZ4_DECOMPRESS_INPLACE_MARGIN(compressedSize) (((compressedSize) >> 8) + 32) +#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize) ((decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize)) /**< note: presumes that compressedSize < decompressedSize. note2: margin is overestimated a bit, since it could use compressedSize instead */ + +#ifndef LZ4_DISTANCE_MAX /* history window size; can be user-defined at compile time */ +# define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */ #endif +#define LZ4_COMPRESS_INPLACE_MARGIN (LZ4_DISTANCE_MAX + 32) /* LZ4_DISTANCE_MAX can be safely replaced by srcSize when it's smaller */ +#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize) ((maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN) /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */ + +#endif /* LZ4_STATIC_3504398509 */ +#endif /* LZ4_STATIC_LINKING_ONLY */ + + + +#ifndef LZ4_H_98237428734687 +#define LZ4_H_98237428734687 /*-************************************************************ * PRIVATE DEFINITIONS @@ -567,6 +639,7 @@ union LZ4_streamDecode_u { } ; /* previously typedef'd to LZ4_streamDecode_t */ + /*-************************************ * Obsolete Functions **************************************/ @@ -674,7 +747,7 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); -#endif /* LZ4_H_2983827168210 */ +#endif /* LZ4_H_98237428734687 */ #if defined (__cplusplus) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 95b8b8e..cc7f2d5 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -1494,7 +1494,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, /* next block is a compressed block */ dctx->tmpInTarget = nextCBlockSize + crcSize; dctx->dStage = dstage_getCBlock; - if (dstPtr==dstEnd) { + if (dstPtr==dstEnd || srcPtr==srcEnd) { nextSrcSizeHint = BHSize + nextCBlockSize + crcSize; doAnotherStage = 0; } diff --git a/lib/lz4hc.h b/lib/lz4hc.h index cdc6d89..44e35bb 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -336,6 +336,9 @@ LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionL #ifndef LZ4_HC_SLO_098092834 #define LZ4_HC_SLO_098092834 +#define LZ4_STATIC_LINKING_ONLY /* LZ4LIB_STATIC_API */ +#include "lz4.h" + #if defined (__cplusplus) extern "C" { #endif diff --git a/tests/fuzzer.c b/tests/fuzzer.c index a5b5c93..b45620b 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -207,7 +207,7 @@ static int FUZ_AddressOverflow(void) } { size_t const sizeToGenerateOverflow = (size_t)(- ((uintptr_t)buffers[nbBuff-1]) + 512); - unsigned const nbOf255 = (unsigned)((sizeToGenerateOverflow / 255) + 1); + int const nbOf255 = (int)((sizeToGenerateOverflow / 255) + 1); char* const input = buffers[nbBuff-1]; char* output = buffers[nbBuff]; int r; @@ -215,7 +215,7 @@ static int FUZ_AddressOverflow(void) input[1] = (char)0xFF; input[2] = (char)0xFF; input[3] = (char)0xFF; - { unsigned u; for(u = 4; u <= nbOf255+4; u++) input[u] = (char)0xff; } + { int u; for(u = 4; u <= nbOf255+4; u++) input[u] = (char)0xff; } r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134); if (r>0) { DISPLAY("LZ4_decompress_safe = %i \n", r); goto _overflowError; } input[0] = (char)0x1F; /* Match length overflow */ @@ -366,7 +366,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c U32 testNb = 0; U32 randState = FUZ_rand(&coreRandState) ^ PRIME3; int const blockSize = (FUZ_rand(&randState) % (FUZ_MAX_BLOCK_SIZE-1)) + 1; - int const blockStart = (FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize - 1)) + 1; + int const blockStart = (int)(FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize - 1)) + 1; int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; int const dictSize = MIN(dictSizeRand, blockStart - 1); int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1); @@ -389,8 +389,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Test compression destSize */ FUZ_DISPLAYTEST("test LZ4_compress_destSize()"); { int srcSize = blockSize; - int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7; - char endCheck = FUZ_rand(&randState) & 255; + int const targetSize = srcSize * (int)((FUZ_rand(&randState) & 127)+1) >> 7; + char endCheck = (char)(FUZ_rand(&randState) & 255); compressedBuffer[targetSize] = endCheck; ret = LZ4_compress_destSize(block, compressedBuffer, &srcSize, targetSize); FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_destSize() result larger than dst buffer !"); @@ -400,7 +400,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c if (targetSize>0) { /* check correctness */ U32 const crcBase = XXH32(block, (size_t)srcSize, 0); - char const canary = FUZ_rand(&randState) & 255; + char const canary = (char)(FUZ_rand(&randState) & 255); FUZ_CHECKTEST((ret==0), "LZ4_compress_destSize() compression failed"); FUZ_DISPLAYTEST(); compressedSize = ret; @@ -409,7 +409,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compress_destSize"); FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data"); FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe() overwrite dst buffer !"); - { U32 const crcDec = XXH32(decodedBuffer, srcSize, 0); + { U32 const crcDec = XXH32(decodedBuffer, (size_t)srcSize, 0); FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); } DISPLAYLEVEL(5, " OK \n"); @@ -420,8 +420,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Test compression HC destSize */ FUZ_DISPLAYTEST("test LZ4_compress_HC_destSize()"); { int srcSize = blockSize; - int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7; - char const endCheck = FUZ_rand(&randState) & 255; + int const targetSize = srcSize * (int)((FUZ_rand(&randState) & 127)+1) >> 7; + char const endCheck = (char)(FUZ_rand(&randState) & 255); void* ctx = LZ4_createHC(block); FUZ_CHECKTEST(ctx==NULL, "LZ4_createHC() allocation failed"); compressedBuffer[targetSize] = endCheck; @@ -435,7 +435,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c if (targetSize>0) { /* check correctness */ U32 const crcBase = XXH32(block, (size_t)srcSize, 0); - char const canary = FUZ_rand(&randState) & 255; + char const canary = (char)(FUZ_rand(&randState) & 255); FUZ_CHECKTEST((ret==0), "LZ4_compress_HC_destSize() compression failed"); FUZ_DISPLAYTEST(); compressedSize = ret; @@ -444,7 +444,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compressHC_destSize"); FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data"); FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe() overwrite dst buffer !"); - { U32 const crcDec = XXH32(decodedBuffer, srcSize, 0); + { U32 const crcDec = XXH32(decodedBuffer, (size_t)srcSize, 0); FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); } DISPLAYLEVEL(5, " OK \n"); @@ -524,7 +524,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(r!=blockSize, "LZ4_decompress_safe did not regenerate original data"); } FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size"); - { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); } @@ -578,7 +578,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Test partial decoding => must work */ FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial"); - { size_t const missingBytes = FUZ_rand(&randState) % blockSize; + { size_t const missingBytes = FUZ_rand(&randState) % (unsigned)blockSize; int const targetSize = (int)((size_t)blockSize - missingBytes); char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A; int const decResult = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, targetSize, blockSize); @@ -705,7 +705,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); - { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); } @@ -792,12 +792,11 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size"); - { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0); if (crcCheck!=crcOrig) { FUZ_findDiff(block, decodedBuffer); EXIT_MSG("LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); - } - } + } } FUZ_DISPLAYTEST(); decodedBuffer[blockSize] = 0; @@ -965,8 +964,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c } -#define testInputSize (192 KB) -#define testCompressedSize (128 KB) +#define testInputSize (196 KB) +#define testCompressedSize (130 KB) #define ringBufferSize (8 KB) static void FUZ_unitTests(int compressionLevel) @@ -990,17 +989,17 @@ static void FUZ_unitTests(int compressionLevel) FUZ_AddressOverflow(); /* Test decoding with empty input */ - DISPLAYLEVEL(3, "LZ4_decompress_safe() with empty input"); + DISPLAYLEVEL(3, "LZ4_decompress_safe() with empty input \n"); LZ4_decompress_safe(testCompressed, testVerify, 0, testInputSize); /* Test decoding with a one byte input */ - DISPLAYLEVEL(3, "LZ4_decompress_safe() with one byte input"); + DISPLAYLEVEL(3, "LZ4_decompress_safe() with one byte input \n"); { char const tmp = (char)0xFF; LZ4_decompress_safe(&tmp, testVerify, 1, testInputSize); } /* Test decoding shortcut edge case */ - DISPLAYLEVEL(3, "LZ4_decompress_safe() with shortcut edge case"); + DISPLAYLEVEL(3, "LZ4_decompress_safe() with shortcut edge case \n"); { char tmp[17]; /* 14 bytes of literals, followed by a 14 byte match. * Should not read beyond the end of the buffer. @@ -1013,6 +1012,54 @@ static void FUZ_unitTests(int compressionLevel) FUZ_CHECKTEST(r >= 0, "LZ4_decompress_safe() should fail"); } } + /* in-place compression test */ + DISPLAYLEVEL(3, "in-place compression using LZ4_compress_default() :"); + { int const sampleSize = 65 KB; + int const maxCSize = LZ4_COMPRESSBOUND(sampleSize); + int const outSize = LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCSize); + int const startInputIndex = outSize - sampleSize; + char* const startInput = testCompressed + startInputIndex; + XXH32_hash_t const crcOrig = XXH32(testInput, sampleSize, 0); + int cSize; + assert(outSize < (int)testCompressedSize); + memcpy(startInput, testInput, sampleSize); /* copy at end of buffer */ + /* compress in-place */ + cSize = LZ4_compress_default(startInput, testCompressed, sampleSize, maxCSize); + assert(cSize != 0); /* ensure compression is successful */ + assert(maxCSize < INT_MAX); + assert(cSize <= maxCSize); + /* decompress and verify */ + { int const dSize = LZ4_decompress_safe(testCompressed, testVerify, cSize, testInputSize); + assert(dSize == sampleSize); /* correct size */ + { XXH32_hash_t const crcCheck = XXH32(testVerify, (size_t)dSize, 0); + assert(crcCheck == crcOrig); + } } } + DISPLAYLEVEL(3, " OK \n"); + + /* in-place decompression test */ + DISPLAYLEVEL(3, "in-place decompression, limit case:"); + { int const sampleSize = 65 KB; + + FUZ_fillCompressibleNoiseBuffer(testInput, sampleSize, 0.0, &randState); + memset(testInput, 0, 267); /* calculated exactly so that compressedSize == originalSize-1 */ + + { XXH64_hash_t const crcOrig = XXH64(testInput, sampleSize, 0); + int const cSize = LZ4_compress_default(testInput, testCompressed, sampleSize, testCompressedSize); + assert(cSize == sampleSize - 1); /* worst case for in-place decompression */ + + { int const bufferSize = LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(sampleSize); + int const startInputIndex = bufferSize - cSize; + char* const startInput = testVerify + startInputIndex; + memcpy(startInput, testCompressed, cSize); + + /* decompress and verify */ + { int const dSize = LZ4_decompress_safe(startInput, testVerify, cSize, sampleSize); + assert(dSize == sampleSize); /* correct size */ + { XXH64_hash_t const crcCheck = XXH64(testVerify, (size_t)dSize, 0); + assert(crcCheck == crcOrig); + } } } } } + DISPLAYLEVEL(3, " OK \n"); + /* LZ4 streaming tests */ { LZ4_stream_t streamingState; U64 crcOrig; @@ -1061,17 +1108,17 @@ static void FUZ_unitTests(int compressionLevel) crcOrig = XXH64_digest(&xxhOrig); memcpy (ringBuffer + rNext, testInput + iNext, messageSize); - compressedSize = LZ4_compress_fast_continue(&streamingState, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize, 1); + compressedSize = LZ4_compress_fast_continue(&streamingState, ringBuffer + rNext, testCompressed, (int)messageSize, testCompressedSize-ringBufferSize, 1); FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_fast_continue() compression failed"); - result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, messageSize); + result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, (int)messageSize); FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe_continue() test failed"); XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize); { U64 const crcNew = XXH64_digest(&xxhNewSafe); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); } - result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, messageSize); + result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, (int)messageSize); FUZ_CHECKTEST(result!=compressedSize, "ringBuffer : LZ4_decompress_fast_continue() test failed"); XXH64_update(&xxhNewFast, testVerify + dNext, messageSize); @@ -1151,12 +1198,12 @@ static void FUZ_unitTests(int compressionLevel) { U64 const crc64 = XXH64(testInput + 64 KB, testCompressedSize, 0); LZ4_resetStreamHC_fast(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, 64 KB); - result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); - FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); - - result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 64 KB); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed"); + { int const cSize = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); + FUZ_CHECKTEST(cSize==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : @return = %i", cSize); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); + { int const dSize = LZ4_decompress_safe_usingDict(testCompressed, testVerify, cSize, testCompressedSize, testInput, 64 KB); + FUZ_CHECKTEST(dSize!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed"); + } } { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() simple dictionary decompression test : corruption"); } } @@ -1165,7 +1212,8 @@ static void FUZ_unitTests(int compressionLevel) /* multiple HC compression test with dictionary */ { int result1, result2; int segSize = testCompressedSize / 2; - U64 const crc64 = XXH64(testInput + segSize, testCompressedSize, 0); + XXH64_hash_t const crc64 = ( (void)assert((unsigned)segSize + testCompressedSize < testInputSize) , + XXH64(testInput + segSize, testCompressedSize, 0) ); LZ4_resetStreamHC_fast(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, segSize); result1 = LZ4_compress_HC_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1); @@ -1179,7 +1227,7 @@ static void FUZ_unitTests(int compressionLevel) FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 1 failed"); result = LZ4_decompress_safe_usingDict(testCompressed+result1, testVerify+segSize, result2, segSize, testInput, 2*segSize); FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 2 failed"); - { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); + { XXH64_hash_t const crcNew = XXH64(testVerify, testCompressedSize, 0); FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); } } @@ -1201,15 +1249,15 @@ static void FUZ_unitTests(int compressionLevel) { XXH64_state_t crcOrigState; XXH64_state_t crcNewState; const char* dict = testInput + 3; - int dictSize = (FUZ_rand(&randState) & 8191); + size_t dictSize = (FUZ_rand(&randState) & 8191); char* dst = testVerify; - size_t segStart = (size_t)dictSize + 7; - int segSize = (FUZ_rand(&randState) & 8191); + size_t segStart = dictSize + 7; + size_t segSize = (FUZ_rand(&randState) & 8191); int segNb = 1; LZ4_resetStreamHC_fast(&sHC, compressionLevel); - LZ4_loadDictHC(&sHC, dict, dictSize); + LZ4_loadDictHC(&sHC, dict, (int)dictSize); XXH64_reset(&crcOrigState, 0); XXH64_reset(&crcNewState, 0); @@ -1217,29 +1265,28 @@ static void FUZ_unitTests(int compressionLevel) while (segStart + segSize < testInputSize) { XXH64_update(&crcOrigState, testInput + segStart, segSize); crcOrig = XXH64_digest(&crcOrigState); - result = LZ4_compress_HC_continue(&sHC, testInput + segStart, testCompressed, segSize, LZ4_compressBound(segSize)); + assert(segSize <= INT_MAX); + result = LZ4_compress_HC_continue(&sHC, testInput + segStart, testCompressed, (int)segSize, LZ4_compressBound((int)segSize)); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); - result = LZ4_decompress_safe_usingDict(testCompressed, dst, result, segSize, dict, dictSize); - FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %i failed", segNb); + result = LZ4_decompress_safe_usingDict(testCompressed, dst, result, (int)segSize, dict, (int)dictSize); + FUZ_CHECKTEST(result!=(int)segSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %i failed", (int)segNb); XXH64_update(&crcNewState, dst, segSize); { U64 const crcNew = XXH64_digest(&crcNewState); if (crcOrig != crcNew) FUZ_findDiff(dst, testInput+segStart); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() part %i corruption", segNb); } - assert(segSize >= 0); dict = dst; dictSize = segSize; - dst += (size_t)segSize + 1; + dst += segSize + 1; segNb ++; - segStart += (size_t)segSize + (FUZ_rand(&randState) & 0xF) + 1; + segStart += segSize + (FUZ_rand(&randState) & 0xF) + 1; segSize = (FUZ_rand(&randState) & 8191); - } - } + } } /* ring buffer test */ { XXH64_state_t xxhOrig; @@ -1266,18 +1313,21 @@ static void FUZ_unitTests(int compressionLevel) crcOrig = XXH64_digest(&xxhOrig); memcpy (ringBuffer + rNext, testInput + iNext, messageSize); - compressedSize = LZ4_compress_HC_continue(&sHC, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); + assert(messageSize < INT_MAX); + compressedSize = LZ4_compress_HC_continue(&sHC, ringBuffer + rNext, testCompressed, (int)messageSize, testCompressedSize-ringBufferSize); FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed"); FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); - result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, messageSize); + assert(messageSize < INT_MAX); + result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, (int)messageSize); FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe_continue() test failed"); XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize); { U64 const crcNew = XXH64_digest(&xxhNewSafe); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); } - result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, messageSize); + assert(messageSize < INT_MAX); + result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, (int)messageSize); FUZ_CHECKTEST(result!=compressedSize, "ringBuffer : LZ4_decompress_fast_continue() test failed"); XXH64_update(&xxhNewFast, testVerify + dNext, messageSize); @@ -1337,14 +1387,14 @@ static void FUZ_unitTests(int compressionLevel) result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, ringBufferSafe + dNext, compressedSize, messageSize); FUZ_CHECKTEST(result!=messageSize, "64K D.ringBuffer : LZ4_decompress_safe_continue() test failed"); - XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, messageSize); + XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, (size_t)messageSize); { U64 const crcNew = XXH64_digest(&xxhNewSafe); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); } result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, ringBufferFast + dNext, messageSize); FUZ_CHECKTEST(result!=compressedSize, "64K D.ringBuffer : LZ4_decompress_fast_continue() test failed"); - XXH64_update(&xxhNewFast, ringBufferFast + dNext, messageSize); + XXH64_update(&xxhNewFast, ringBufferFast + dNext, (size_t)messageSize); { U64 const crcNew = XXH64_digest(&xxhNewFast); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption"); } @@ -1360,7 +1410,7 @@ static void FUZ_unitTests(int compressionLevel) dNext = 0; while (totalMessageSize < 9 MB) { - XXH64_update(&xxhOrig, testInput + iNext, messageSize); + XXH64_update(&xxhOrig, testInput + iNext, (size_t)messageSize); crcOrig = XXH64_digest(&xxhOrig); compressedSize = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); @@ -1375,7 +1425,7 @@ static void FUZ_unitTests(int compressionLevel) testCompressed, ringBufferSafe + dNext, compressedSize, dBufferSize - dNext); /* works without knowing messageSize, under assumption that messageSize <= maxMessageSize */ FUZ_CHECKTEST(result!=messageSize, "D.ringBuffer : LZ4_decompress_safe_continue() test failed"); - XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, messageSize); + XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, (size_t)messageSize); { U64 const crcNew = XXH64_digest(&xxhNewSafe); if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, ringBufferSafe + dNext); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption during D.ringBuffer test"); @@ -1384,7 +1434,7 @@ static void FUZ_unitTests(int compressionLevel) /* test LZ4_decompress_fast_continue in its own buffer ringBufferFast */ result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, ringBufferFast + dNext, messageSize); FUZ_CHECKTEST(result!=compressedSize, "D.ringBuffer : LZ4_decompress_fast_continue() test failed"); - XXH64_update(&xxhNewFast, ringBufferFast + dNext, messageSize); + XXH64_update(&xxhNewFast, ringBufferFast + dNext, (size_t)messageSize); { U64 const crcNew = XXH64_digest(&xxhNewFast); if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, ringBufferFast + dNext); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption during D.ringBuffer test"); @@ -1392,7 +1442,8 @@ static void FUZ_unitTests(int compressionLevel) /* prepare next message */ dNext += messageSize; - totalMessageSize += messageSize; + assert(messageSize >= 0); + totalMessageSize += (unsigned)messageSize; messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1; iNext = (FUZ_rand(&randState) & 65535); if (dNext + maxMessageSize > dBufferSize) dNext = 0; @@ -1410,6 +1461,11 @@ static void FUZ_unitTests(int compressionLevel) } + +/* ======================================= + * CLI + * ======================================= */ + static int FUZ_usage(const char* programName) { DISPLAY( "Usage :\n"); |