diff options
author | Yann Collet <Cyan4973@users.noreply.github.com> | 2018-04-18 17:16:25 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-04-18 17:16:25 (GMT) |
commit | c6301479c5a85aa32d6060b3976995c1ad85a58f (patch) | |
tree | 24b53a0b402c8e34955f2321b98036af3bdc37a7 | |
parent | a35aba6d747854f8b0d7137c7d3dbc67f6a9fb33 (diff) | |
parent | 5ad4599c5ad18d2408a6ccc545c45a36b99f0c6f (diff) | |
download | lz4-c6301479c5a85aa32d6060b3976995c1ad85a58f.zip lz4-c6301479c5a85aa32d6060b3976995c1ad85a58f.tar.gz lz4-c6301479c5a85aa32d6060b3976995c1ad85a58f.tar.bz2 |
Merge pull request #497 from lz4/lowAddr
Compatibility with low memory addresses
-rw-r--r-- | .travis.yml | 6 | ||||
-rw-r--r-- | doc/lz4_manual.html | 94 | ||||
-rw-r--r-- | lib/lz4.c | 299 | ||||
-rw-r--r-- | lib/lz4hc.c | 16 | ||||
-rw-r--r-- | lib/lz4hc.h | 8 | ||||
-rw-r--r-- | tests/Makefile | 3 | ||||
-rw-r--r-- | tests/fuzzer.c | 246 |
7 files changed, 443 insertions, 229 deletions
diff --git a/.travis.yml b/.travis.yml index 466d55e..0a876f9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,8 +14,8 @@ matrix: env: Ubu=12.04cont Cmd='make -C tests test-lz4 test-lz4c test-fullbench' COMPILER=cc - os: linux - sudo: false - env: Ubu=12.04cont Cmd='make -C tests test-frametest test-fuzzer' COMPILER=cc + sudo: required + env: Ubu=12.04cont Cmd='sudo sysctl -w vm.mmap_min_addr="4096" && make -C tests test-frametest test-fuzzer' COMPILER=cc - os: linux sudo: false @@ -59,7 +59,7 @@ matrix: - libc6-dev-i386 - gcc-multilib - - env: Ubu=14.04 Cmd='make -C tests test-frametest32 test-fuzzer32' COMPILER=cc + - env: Ubu=14.04 Cmd='sudo sysctl -w vm.mmap_min_addr="4096" && make -C tests test-frametest32 test-fuzzer32' COMPILER=cc dist: trusty sudo: required addons: diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html index d14467f..f8639fe 100644 --- a/doc/lz4_manual.html +++ b/doc/lz4_manual.html @@ -15,8 +15,9 @@ <li><a href="#Chapter5">Advanced Functions</a></li> <li><a href="#Chapter6">Streaming Compression Functions</a></li> <li><a href="#Chapter7">Streaming Decompression Functions</a></li> -<li><a href="#Chapter8">Private definitions</a></li> -<li><a href="#Chapter9">Obsolete Functions</a></li> +<li><a href="#Chapter8">Unstable declarations</a></li> +<li><a href="#Chapter9">Private definitions</a></li> +<li><a href="#Chapter10">Obsolete Functions</a></li> </ol> <hr> <a name="Chapter1"></a><h2>Introduction</h2><pre> @@ -245,22 +246,80 @@ int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, </p></pre><BR> -<a name="Chapter8"></a><h2>Private definitions</h2><pre> +<a name="Chapter8"></a><h2>Unstable declarations</h2><pre> + Declarations in this section should be considered unstable. + Use at your own peril, etc., etc. + They may be removed in the future. + Their signatures may change. +<BR></pre> + +<pre><b>void LZ4_resetStream_fast (LZ4_stream_t* streamPtr); +</b><p> When an LZ4_stream_t is known to be in a internally coherent state, + it can often be prepared for a new compression with almost no work, only + sometimes falling back to the full, expensive reset that is always required + when the stream is in an indeterminate state (i.e., the reset performed by + LZ4_resetStream()). + + LZ4_streams are guaranteed to be in a valid state when: + - returned from LZ4_createStream() + - reset by LZ4_resetStream() + - memset(stream, 0, sizeof(LZ4_stream_t)) + - the stream was in a valid state and was reset by LZ4_resetStream_fast() + - the stream was in a valid state and was then used in any compression call + that returned success + - the stream was in an indeterminate state and was used in a compression + call that fully reset the state (LZ4_compress_fast_extState()) and that + returned success + +</p></pre><BR> + +<pre><b>int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); +</b><p> A variant of LZ4_compress_fast_extState(). + + Using this variant avoids an expensive initialization step. It is only safe + to call if the state buffer is known to be correctly initialized already + (see above comment on LZ4_resetStream_fast() for a definition of "correctly + initialized"). From a high level, the difference is that this function + initializes the provided state with a call to LZ4_resetStream_fast() while + LZ4_compress_fast_extState() starts with a call to LZ4_resetStream(). + +</p></pre><BR> + +<pre><b>void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream); +</b><p> This is an experimental API that allows for the efficient use of a + static dictionary many times. + + Rather than re-loading the dictionary buffer into a working context before + each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a + working LZ4_stream_t, this function introduces a no-copy setup mechanism, + in which the working stream references the dictionary stream in-place. + + Several assumptions are made about the state of the dictionary stream. + Currently, only streams which have been prepared by LZ4_loadDict() should + be expected to work. + + Alternatively, the provided dictionary stream pointer may be NULL, in which + case any existing dictionary stream is unset. + + If a dictionary is provided, it replaces any pre-existing stream history. + The dictionary contents are the only history that can be referenced and + logically immediately precede the data compressed in the first subsequent + compression call. + + The dictionary will only remain attached to the working stream through the + first compression call, at the end of which it is cleared. The dictionary + stream (and source buffer) must remain in-place / accessible / unchanged + through the completion of the first compression call on the stream. + +</p></pre><BR> + +<a name="Chapter9"></a><h2>Private definitions</h2><pre> Do not use these definitions. They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. Using these definitions will expose code to API and/or ABI break in future versions of the library. <BR></pre> <pre><b>typedef struct { - uint32_t hashTable[LZ4_HASH_SIZE_U32]; - uint32_t currentOffset; - uint32_t initCheck; - const uint8_t* dictionary; - uint8_t* bufferStart; </b>/* obsolete, used for slideInputBuffer */<b> - uint32_t dictSize; -} LZ4_stream_t_internal; -</b></pre><BR> -<pre><b>typedef struct { const uint8_t* externalDict; size_t extDictSize; const uint8_t* prefixEnd; @@ -268,15 +327,6 @@ int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, } LZ4_streamDecode_t_internal; </b></pre><BR> <pre><b>typedef struct { - unsigned int hashTable[LZ4_HASH_SIZE_U32]; - unsigned int currentOffset; - unsigned int initCheck; - const unsigned char* dictionary; - unsigned char* bufferStart; </b>/* obsolete, used for slideInputBuffer */<b> - unsigned int dictSize; -} LZ4_stream_t_internal; -</b></pre><BR> -<pre><b>typedef struct { const unsigned char* externalDict; size_t extDictSize; const unsigned char* prefixEnd; @@ -311,7 +361,7 @@ union LZ4_streamDecode_u { </p></pre><BR> -<a name="Chapter9"></a><h2>Obsolete Functions</h2><pre></pre> +<a name="Chapter10"></a><h2>Obsolete Functions</h2><pre></pre> <pre><b>#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS # define LZ4_DEPRECATED(message) </b>/* disable deprecation warnings */<b> @@ -517,6 +517,18 @@ LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tab return LZ4_hash4(LZ4_read32(p), tableType); } +static void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t const tableType) +{ + switch (tableType) + { + default: /* fallthrough */ + case clearedTable: /* fallthrough */ + case byPtr: { /* illegal! */ assert(0); return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = idx; return; } + case byU16: { U16* hashTable = (U16*) tableBase; assert(idx < 65536); hashTable[h] = (U16)idx; return; } + } +} + static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) { switch (tableType) @@ -534,6 +546,20 @@ LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_ LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); } +/* LZ4_getIndexOnHash() : + * Index of match position registered in hash table. + * hash position must be calculated by using base+index, or dictBase+index. + * Assumption 1 : only valid if tableType == byU32 or byU16. + * Assumption 2 : h is presumed valid (within limits of hash table) + */ +static U32 LZ4_getIndexOnHash(U32 h, const void* tableBase, tableType_t tableType) +{ + LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2); + if (tableType == byU32) { const U32* const hashTable = (const U32*) tableBase; assert(h < (1U << (LZ4_MEMORY_USAGE-2))); return hashTable[h]; } + if (tableType == byU16) { const U16* const hashTable = (const U16*) tableBase; assert(h < (1U << (LZ4_MEMORY_USAGE-1))); return hashTable[h]; } + assert(0); return 0; /* forbidden case */ +} + static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType_t tableType, const BYTE* srcBase) { if (tableType == byPtr) { const BYTE* const* hashTable = (const BYTE* const*) tableBase; return hashTable[h]; } @@ -562,10 +588,12 @@ LZ4_FORCE_INLINE void LZ4_prepareTable( || tableType == byPtr || inputSize >= 4 KB) { - DEBUGLOG(4, "Resetting table in %p", cctx); + DEBUGLOG(4, "LZ4_prepareTable: Resetting table in %p", cctx); MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE); cctx->currentOffset = 0; cctx->tableType = clearedTable; + } else { + DEBUGLOG(4, "LZ4_prepareTable: Re-use hash table (no reset)"); } } @@ -574,6 +602,7 @@ LZ4_FORCE_INLINE void LZ4_prepareTable( * currentOffset == 0 is faster still, so we preserve that case. */ if (cctx->currentOffset != 0 && tableType == byU32) { + DEBUGLOG(5, "LZ4_prepareTable: adding 64KB to currentOffset"); cctx->currentOffset += 64 KB; } @@ -599,8 +628,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( { const BYTE* ip = (const BYTE*) source; - size_t currentOffset = cctx->currentOffset; - const BYTE* base = (const BYTE*) source - currentOffset; + U32 const startIndex = cctx->currentOffset; + const BYTE* base = (const BYTE*) source - startIndex; const BYTE* lowLimit; const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx; @@ -608,8 +637,10 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( dictDirective == usingDictCtx ? dictCtx->dictionary : cctx->dictionary; const U32 dictSize = dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize; + const U32 dictDelta = (dictDirective == usingDictCtx) ? startIndex - dictCtx->currentOffset : 0; /* make indexes in dictCtx comparable with index in current context */ - const BYTE* const lowRefLimit = (const BYTE*) source - dictSize; + int const maybe_extMem = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx); + U32 const prefixIdxLimit = startIndex - dictSize; /* used when dictDirective == dictSmall */ const BYTE* const dictEnd = dictionary + dictSize; const BYTE* anchor = (const BYTE*) source; const BYTE* const iend = ip + inputSize; @@ -619,21 +650,22 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( /* the dictCtx currentOffset is indexed on the start of the dictionary, * while a dictionary in the current context precedes the currentOffset */ const BYTE* dictBase = dictDirective == usingDictCtx ? - (const BYTE*) source - dictCtx->currentOffset : - (const BYTE*) source - dictSize - currentOffset; - const ptrdiff_t dictDelta = dictionary ? dictEnd - (const BYTE*) source : 0; - const BYTE* dictLowLimit; + dictionary + dictSize - dictCtx->currentOffset : /* is it possible that dictCtx->currentOffset != dictCtx->dictSize ? Yes if the dictionary context is not reset */ + dictionary + dictSize - startIndex; BYTE* op = (BYTE*) dest; BYTE* const olimit = op + maxOutputSize; + U32 offset = 0; U32 forwardH; + DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, tableType=%u", inputSize, tableType); /* Init conditions */ if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */ + if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */ + assert(acceleration >= 1); lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0); - dictLowLimit = dictionary ? dictionary : lowLimit; if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ @@ -657,12 +689,12 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( /* Main Loop */ for ( ; ; ) { - ptrdiff_t refDelta = 0; const BYTE* match; BYTE* token; /* Find a match */ - { const BYTE* forwardIp = ip; + if (tableType == byPtr) { + const BYTE* forwardIp = ip; unsigned step = 1; unsigned searchMatchNb = acceleration << LZ4_skipTrigger; do { @@ -675,34 +707,71 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( assert(ip < mflimitPlusOne); match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base); + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); + + } while ( (match+MAX_DISTANCE < ip) + || (LZ4_read32(match) != LZ4_read32(ip)) ); + + } else { /* byU32, byU16 */ + + const BYTE* forwardIp = ip; + unsigned step = 1; + unsigned searchMatchNb = acceleration << LZ4_skipTrigger; + do { + U32 const h = forwardH; + U32 const current = (U32)(forwardIp - base); + U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + assert(matchIndex <= current); + assert(forwardIp - base < (ptrdiff_t)(2 GB - 1)); + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; + assert(ip < mflimitPlusOne); + if (dictDirective == usingDictCtx) { - if (match < (const BYTE*)source) { + if (matchIndex < startIndex) { /* there was no match, try the dictionary */ - match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase); - refDelta = dictDelta; - lowLimit = dictLowLimit; + assert(tableType == byU32); + matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); + match = dictBase + matchIndex; + matchIndex += dictDelta; /* make dictCtx index comparable with current context */ + lowLimit = dictionary; } else { - refDelta = 0; + match = base + matchIndex; lowLimit = (const BYTE*)source; } } else if (dictDirective==usingExtDict) { - if (match < (const BYTE*)source) { - refDelta = dictDelta; - lowLimit = dictLowLimit; + if (matchIndex < startIndex) { + DEBUGLOG(7, "extDict candidate: matchIndex=%5u < startIndex=%5u", matchIndex, startIndex); + match = dictBase + matchIndex; + lowLimit = dictionary; } else { - refDelta = 0; + match = base + matchIndex; lowLimit = (const BYTE*)source; - } } + } + } else { /* single continuous memory segment */ + match = base + matchIndex; + } forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); + LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); + + if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue; /* match outside of valid area */ + if ((tableType != byU16) && (current - matchIndex > MAX_DISTANCE)) continue; /* too far - note: works even if matchIndex overflows */ + if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE); /* too_far presumed impossible with byU16 */ + + if (LZ4_read32(match) == LZ4_read32(ip)) { + if (maybe_extMem) offset = current - matchIndex; + break; /* match found */ + } - } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) - || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) - || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); + } while(1); } /* Catch up */ - while (((ip>anchor) & (match+refDelta > lowLimit)) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } + while (((ip>anchor) & (match > lowLimit)) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } /* Encode Literals */ { unsigned const litLength = (unsigned)(ip - anchor); @@ -721,30 +790,49 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( /* Copy Literals */ LZ4_wildCopy(op, anchor, op+litLength); op+=litLength; + DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", (int)(anchor-(const BYTE*)source), litLength, (int)(ip-(const BYTE*)source)); } _next_match: + /* at this stage, the following variables must be correctly set : + * - ip : at start of LZ operation + * - match : at start of previous pattern occurence; can be within current prefix, or within extDict + * - offset : if maybe_ext_memSegment==1 (constant) + * - lowLimit : must be == dictionary to mean "match is within extDict"; must be == source otherwise + * - token and *token : position to write 4-bits for match length; higher 4-bits for literal length supposed already written + */ + /* Encode Offset */ - LZ4_writeLE16(op, (U16)(ip-match)); op+=2; + if (maybe_extMem) { /* static test */ + DEBUGLOG(6, " with offset=%u (ext if > %i)", offset, (int)(ip - (const BYTE*)source)); + assert(offset <= MAX_DISTANCE && offset > 0); + LZ4_writeLE16(op, (U16)offset); op+=2; + } else { + DEBUGLOG(6, " with offset=%u (same segment)", (U32)(ip - match)); + assert(ip-match <= MAX_DISTANCE); + LZ4_writeLE16(op, (U16)(ip - match)); op+=2; + } /* Encode MatchLength */ { unsigned matchCode; - if ((dictDirective==usingExtDict || dictDirective==usingDictCtx) && lowLimit==dictionary) { - const BYTE* limit; - match += refDelta; - limit = ip + (dictEnd-match); + if ( (dictDirective==usingExtDict || dictDirective==usingDictCtx) + && (lowLimit==dictionary) /* match within extDict */ ) { + const BYTE* limit = ip + (dictEnd-match); + assert(dictEnd > match); if (limit > matchlimit) limit = matchlimit; matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); ip += MINMATCH + matchCode; if (ip==limit) { - unsigned const more = LZ4_count(ip, (const BYTE*)source, matchlimit); + unsigned const more = LZ4_count(limit, (const BYTE*)source, matchlimit); matchCode += more; ip += more; } + DEBUGLOG(6, " with matchLength=%u starting in extDict", matchCode+MINMATCH); } else { matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); ip += MINMATCH + matchCode; + DEBUGLOG(6, " with matchLength=%u", matchCode+MINMATCH); } if ( outputLimited && /* Check output buffer overflow */ @@ -774,33 +862,57 @@ _next_match: LZ4_putPosition(ip-2, cctx->hashTable, tableType, base); /* Test next position */ - match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); - if (dictDirective == usingDictCtx) { - if (match < (const BYTE*)source) { - /* there was no match, try the dictionary */ - match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase); - refDelta = dictDelta; - lowLimit = dictLowLimit; - } else { - refDelta = 0; - lowLimit = (const BYTE*)source; + if (tableType == byPtr) { + + match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); + LZ4_putPosition(ip, cctx->hashTable, tableType, base); + if ( (match+MAX_DISTANCE >= ip) + && (LZ4_read32(match) == LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + } else { /* byU32, byU16 */ + + U32 const h = LZ4_hashPosition(ip, tableType); + U32 const current = (U32)(ip-base); + U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + assert(matchIndex < current); + if (dictDirective == usingDictCtx) { + if (matchIndex < startIndex) { + /* there was no match, try the dictionary */ + matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); + match = dictBase + matchIndex; + lowLimit = dictionary; /* required for match length counter */ + matchIndex += dictDelta; + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; /* required for match length counter */ + } + } else if (dictDirective==usingExtDict) { + if (matchIndex < startIndex) { + match = dictBase + matchIndex; + lowLimit = dictionary; /* required for match length counter */ + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; /* required for match length counter */ + } + } else { /* single memory segment */ + match = base + matchIndex; } - } else if (dictDirective==usingExtDict) { - if (match < (const BYTE*)source) { - refDelta = dictDelta; - lowLimit = dictLowLimit; - } else { - refDelta = 0; - lowLimit = (const BYTE*)source; - } } - LZ4_putPosition(ip, cctx->hashTable, tableType, base); - if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) - && (match+MAX_DISTANCE>=ip) - && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) - { token=op++; *token=0; goto _next_match; } + LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); + if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1) + && ((tableType==byU16) ? 1 : (current - matchIndex <= MAX_DISTANCE)) + && (LZ4_read32(match) == LZ4_read32(ip)) ) { + token=op++; + *token=0; + if (maybe_extMem) offset = current - matchIndex; + DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source)); + goto _next_match; + } + } /* Prepare next loop */ forwardH = LZ4_hashPosition(++ip, tableType); + } _last_literals: @@ -834,14 +946,14 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int if (inputSize < LZ4_64Klimit) { return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); } else { - const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr; + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > MAX_DISTANCE)) ? byPtr : byU32; return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration); } } else { if (inputSize < LZ4_64Klimit) {; return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); } else { - const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr; + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > MAX_DISTANCE)) ? byPtr : byU32; return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration); } } @@ -856,38 +968,38 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int * (see comment in lz4.h on LZ4_resetStream_fast() for a definition of * "correctly initialized"). */ -int LZ4_compress_fast_extState_fastReset(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration) { LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse; if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; - if (maxOutputSize >= LZ4_compressBound(inputSize)) { - if (inputSize < LZ4_64Klimit) { + if (dstCapacity >= LZ4_compressBound(srcSize)) { + if (srcSize < LZ4_64Klimit) { const tableType_t tableType = byU16; - LZ4_prepareTable(ctx, inputSize, tableType); + LZ4_prepareTable(ctx, srcSize, tableType); if (ctx->currentOffset) { - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, dictSmall, acceleration); + return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, dictSmall, acceleration); } else { - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration); } } else { - const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr; - LZ4_prepareTable(ctx, inputSize, tableType); - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32; + LZ4_prepareTable(ctx, srcSize, tableType); + return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration); } } else { - if (inputSize < LZ4_64Klimit) { + if (srcSize < LZ4_64Klimit) { const tableType_t tableType = byU16; - LZ4_prepareTable(ctx, inputSize, tableType); + LZ4_prepareTable(ctx, srcSize, tableType); if (ctx->currentOffset) { - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration); + return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration); } else { - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); } } else { - const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr; - LZ4_prepareTable(ctx, inputSize, tableType); - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration); + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32; + LZ4_prepareTable(ctx, srcSize, tableType); + return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); } } } @@ -1099,11 +1211,12 @@ static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src, if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); } else { - if (*srcSizePtr < LZ4_64Klimit) + if (*srcSizePtr < LZ4_64Klimit) { return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16); - else - return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, sizeof(void*)==8 ? byU32 : byPtr); - } + } else { + tableType_t const tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32; + return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, tableType); + } } } @@ -1143,7 +1256,7 @@ LZ4_stream_t* LZ4_createStream(void) void LZ4_resetStream (LZ4_stream_t* LZ4_stream) { - DEBUGLOG(5, "LZ4_resetStream %p", LZ4_stream); + DEBUGLOG(5, "LZ4_resetStream (ctx:%p)", LZ4_stream); MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); } @@ -1169,7 +1282,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) const BYTE* const dictEnd = p + dictSize; const BYTE* base; - DEBUGLOG(4, "LZ4_loadDict %p", LZ4_dict); + DEBUGLOG(4, "LZ4_loadDict (%i bytes from %p into %p)", dictSize, dictionary, LZ4_dict); LZ4_prepareTable(dict, 0, tableType); @@ -1216,15 +1329,14 @@ void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dic } -static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) +static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, int nextSize) { - if ((LZ4_dict->currentOffset > 0x80000000) || - ((uptrval)LZ4_dict->currentOffset > (uptrval)src)) { /* address space overflow */ + if (LZ4_dict->currentOffset + nextSize > 0x80000000) { /* potential ptrdiff_t overflow (32-bits mode) */ /* rescale hash table */ U32 const delta = LZ4_dict->currentOffset - 64 KB; const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; int i; - DEBUGLOG(4, "LZ4_renormDictT %p", LZ4_dict); + DEBUGLOG(4, "LZ4_renormDictT"); for (i=0; i<LZ4_HASH_SIZE_U32; i++) { if (LZ4_dict->hashTable[i] < delta) LZ4_dict->hashTable[i]=0; else LZ4_dict->hashTable[i] -= delta; @@ -1242,10 +1354,8 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse; const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; - const BYTE* smallest = (const BYTE*) source; if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ - if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; - LZ4_renormDictT(streamPtr, smallest); + LZ4_renormDictT(streamPtr, inputSize); /* avoid index overflow */ if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; /* Check overlapping input/dictionary space */ @@ -1278,7 +1388,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch if (inputSize > 4 KB) { /* For compressing large blobs, it is faster to pay the setup * cost to copy the dictionary's tables into the active context, - * so that the compression loop is only looking in one table. + * so that the compression loop is only looking into one table. */ memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t)); result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); @@ -1299,25 +1409,22 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch } -/* Hidden debug function, to force external dictionary mode */ -int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) +/* Hidden debug function, to force-test external dictionary mode */ +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize) { LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse; int result; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; - const BYTE* smallest = dictEnd; - if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; - LZ4_renormDictT(streamPtr, smallest); + LZ4_renormDictT(streamPtr, srcSize); if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, dictSmall, 1); + result = LZ4_compress_generic(streamPtr, source, dest, srcSize, 0, notLimited, byU32, usingExtDict, dictSmall, 1); } else { - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); + result = LZ4_compress_generic(streamPtr, source, dest, srcSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); } streamPtr->dictionary = (const BYTE*)source; - streamPtr->dictSize = (U32)inputSize; + streamPtr->dictSize = (U32)srcSize; return result; } diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 0c1b20a..111a09b 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -336,7 +336,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( U32 const mlAdd = (matchLength>=19) ? ((matchLength-19) / 255) + 1 : 0; U32 const cost = 1 + llAdd + ll + 2 + mlAdd; if (start==NULL) start = *anchor; /* only works for single segment */ - //g_debuglog_enable = (pos >= 2228) & (pos <= 2262); + /* g_debuglog_enable = (pos >= 2228) & (pos <= 2262); */ DEBUGLOG(6, "pos:%7u -- literals:%3u, match:%4i, offset:%5u, cost:%3u + %u", pos, (U32)(*ip - *anchor), matchLength, (U32)(*ip-match), @@ -852,16 +852,17 @@ int LZ4_resetStreamStateHC(void* state, char* inputBuffer) LZ4HC_CCtx_internal *ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */ LZ4HC_init(ctx, (const BYTE*)inputBuffer); - ctx->inputBuffer = (BYTE*)inputBuffer; + ctx->inputBuffer = inputBuffer; return 0; } -void* LZ4_createHC (char* inputBuffer) +void* LZ4_createHC (const char* inputBuffer) { LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t)); if (hc4 == NULL) return NULL; /* not enough memory */ LZ4HC_init (&hc4->internal_donotuse, (const BYTE*)inputBuffer); - hc4->internal_donotuse.inputBuffer = (BYTE*)inputBuffer; + assert(sizeof(size_t) == sizeof(void*)); + hc4->internal_donotuse.inputBuffer = (void*)(size_t)inputBuffer; /* ugly hack, circumvent -Wcast-qual */ return hc4; } @@ -885,7 +886,7 @@ char* LZ4_slideInputBufferHC(void* LZ4HC_Data) { LZ4HC_CCtx_internal* const hc4 = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse; int const dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB); - return (char*)(hc4->inputBuffer + dictSize); + return (char*)(hc4->inputBuffer) + dictSize; } @@ -1136,15 +1137,14 @@ static int LZ4HC_compress_optimal ( encode: /* cur, last_match_pos, best_mlen, best_off must be set */ assert(cur < LZ4_OPT_NUM); assert(last_match_pos >= 1); /* == 1 when only one candidate */ - DEBUGLOG(6, "reverse traversal, looking for shortest path") - DEBUGLOG(6, "last_match_pos = %i", last_match_pos); + DEBUGLOG(6, "reverse traversal, looking for shortest path (last_match_pos=%i)", last_match_pos); { int candidate_pos = cur; int selected_matchLength = best_mlen; int selected_offset = best_off; while (1) { /* from end to beginning */ int const next_matchLength = opt[candidate_pos].mlen; /* can be 1, means literal */ int const next_offset = opt[candidate_pos].off; - DEBUGLOG(6, "pos %i: sequence length %i", candidate_pos, selected_matchLength); + DEBUGLOG(7, "pos %i: sequence length %i", candidate_pos, selected_matchLength); opt[candidate_pos].mlen = selected_matchLength; opt[candidate_pos].off = selected_offset; selected_matchLength = next_matchLength; diff --git a/lib/lz4hc.h b/lib/lz4hc.h index a7f77f9..7a25bee 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -148,7 +148,7 @@ typedef struct const uint8_t* end; /* next block here to continue on current prefix */ const uint8_t* base; /* All index relative to this position */ const uint8_t* dictBase; /* alternate base for extDict */ - uint8_t* inputBuffer; /* deprecated */ + void* inputBuffer; /* deprecated */ uint32_t dictLimit; /* below that point, need extDict */ uint32_t lowLimit; /* below that point, no more dict */ uint32_t nextToUpdate; /* index from which to continue dictionary update */ @@ -164,7 +164,7 @@ typedef struct const unsigned char* end; /* next block here to continue on current prefix */ const unsigned char* base; /* All index relative to this position */ const unsigned char* dictBase; /* alternate base for extDict */ - unsigned char* inputBuffer; /* deprecated */ + void* inputBuffer; /* deprecated */ unsigned int dictLimit; /* below that point, need extDict */ unsigned int lowLimit; /* below that point, no more dict */ unsigned int nextToUpdate; /* index from which to continue dictionary update */ @@ -206,8 +206,8 @@ LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_co LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize); LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); -/* Deprecated Streaming functions using older model; should no longer be used */ -LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (char* inputBuffer); +/* Deprecated Streaming functions; should no longer be used */ +LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (const char* inputBuffer); LZ4_DEPRECATED("use LZ4_saveDictHC() instead") LZ4LIB_API char* LZ4_slideInputBufferHC (void* LZ4HC_Data); LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") LZ4LIB_API int LZ4_freeHC (void* LZ4HC_Data); LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); diff --git a/tests/Makefile b/tests/Makefile index 25f00b8..2b93c9f 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -33,7 +33,8 @@ PRGDIR := ../programs TESTDIR := versionsTest PYTHON ?= python3 -DEBUGFLAGS = -g -DLZ4_DEBUG=1 +DEBUGLEVEL?= 1 +DEBUGFLAGS = -g -DLZ4_DEBUG=$(DEBUGLEVEL) CFLAGS ?= -O3 # can select custom optimization flags. For example : CFLAGS=-O2 make CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \ -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \ diff --git a/tests/fuzzer.c b/tests/fuzzer.c index db05738..244cc4f 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -34,9 +34,13 @@ #define LZ4_DISABLE_DEPRECATE_WARNINGS + /*-************************************ * Dependencies **************************************/ +#ifdef __unix__ /* must be included before platform.h for MAP_ANONYMOUS */ +# include <sys/mman.h> /* mmap */ +#endif #include "platform.h" /* _CRT_SECURE_NO_WARNINGS */ #include "util.h" /* U32 */ #include <stdlib.h> @@ -241,6 +245,41 @@ _overflowError: } +#ifdef __unix__ /* is expected to be triggered on linux+gcc */ + +static void* FUZ_createLowAddr(size_t size) +{ + void* const lowBuff = mmap((void*)(0x1000), size, + PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0); + DISPLAYLEVEL(2, "generating low buffer at address %p \n", lowBuff); + return lowBuff; +} + +static void FUZ_freeLowAddr(void* buffer, size_t size) +{ + if (munmap(buffer, size)) { + perror("fuzzer: freeing low address buffer"); + abort(); + } +} + +#else + +static void* FUZ_createLowAddr(size_t size) +{ + return malloc(size); +} + +static void FUZ_freeLowAddr(void* buffer, size_t size) +{ + (void)size; + free(buffer); +} + +#endif + + /*! FUZ_findDiff() : * find the first different byte between buff1 and buff2. * presumes buff1 != buff2. @@ -253,7 +292,7 @@ static void FUZ_findDiff(const void* buff1, const void* buff2) const BYTE* const b2 = (const BYTE*)buff2; size_t u = 0; while (b1[u]==b2[u]) u++; - DISPLAY("Wrong Byte at position %u \n", (unsigned)u); + DISPLAY("\nWrong Byte at position %u \n", (unsigned)u); } @@ -267,6 +306,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c size_t const compressedBufferSize = LZ4_compressBound(FUZ_MAX_BLOCK_SIZE); char* const compressedBuffer = (char*)malloc(compressedBufferSize); char* const decodedBuffer = (char*)malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE); + size_t const labSize = 96 KB; + void* const lowAddrBuffer = FUZ_createLowAddr(labSize); void* const stateLZ4 = malloc(LZ4_sizeofState()); void* const stateLZ4HC = malloc(LZ4_sizeofStateHC()); LZ4_stream_t LZ4dict; @@ -279,7 +320,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c # define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \ printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; } -# define FUZ_DISPLAYTEST { testNb++; g_displayLevel>=4 ? printf("%2u\b\b", testNb), fflush(stdout) : 0; } +# define FUZ_DISPLAYTEST(...) { \ + testNb++; \ + if (g_displayLevel>=4) { \ + printf("\r%4u - %2u ", cycleNb, testNb); \ + printf(" " __VA_ARGS__); \ + printf(" "); \ + fflush(stdout); \ + } } /* init */ @@ -307,20 +355,24 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; int const dictSize = MIN(dictSizeRand, blockStart - 1); int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1); - char* const block = ((char*)CNBuffer) + blockStart; + const char* block = ((char*)CNBuffer) + blockStart; const char* dict = block - dictSize; int compressedSize, HCcompressedSize; int blockContinueCompressedSize; U32 const crcOrig = XXH32(block, blockSize, 0); - U32 crcCheck; int ret; FUZ_displayUpdate(cycleNb); /* Compression tests */ + if ( ((FUZ_rand(&randState) & 63) == 2) + && ((size_t)blockSize < labSize) ) { + memcpy(lowAddrBuffer, block, blockSize); + block = (const char*)lowAddrBuffer; + } /* Test compression destSize */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_destSize()"); { int srcSize = blockSize; int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7; char endCheck = FUZ_rand(&randState) & 255; @@ -335,7 +387,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c U32 const crcBase = XXH32(block, srcSize, 0); char const canary = FUZ_rand(&randState) & 255; FUZ_CHECKTEST((ret==0), "LZ4_compress_destSize() compression failed"); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); compressedSize = ret; decodedBuffer[srcSize] = canary; ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize); @@ -351,7 +403,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c } } /* Test compression HC destSize */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_HC_destSize()"); { int srcSize = blockSize; int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7; char const endCheck = FUZ_rand(&randState) & 255; @@ -365,14 +417,12 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_HC_destSize() result larger than dst buffer !"); FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_HC_destSize() overwrite dst buffer !"); FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_HC_destSize() fed more than src buffer !"); - DISPLAYLEVEL(5, "LZ4_compress_HC_destSize(%i): destSize : %7i/%7i; content%7i/%7i ", - compressionLevel, ret, targetSize, srcSize, blockSize); if (targetSize>0) { /* check correctness */ U32 const crcBase = XXH32(block, srcSize, 0); char const canary = FUZ_rand(&randState) & 255; FUZ_CHECKTEST((ret==0), "LZ4_compress_HC_destSize() compression failed"); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); compressedSize = ret; decodedBuffer[srcSize] = canary; ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize); @@ -388,66 +438,68 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c } } /* Test compression HC */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_HC()"); ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel); FUZ_CHECKTEST(ret==0, "LZ4_compress_HC() failed"); HCcompressedSize = ret; /* Test compression HC using external state */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC()"); ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel); FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed"); /* Test compression using external state */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_fast_extState()"); ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8); FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed"); /* Test compression using fast reset external state*/ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); ret = LZ4_compress_fast_extState_fastReset(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8); FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState_fastReset() failed"); /* Test compression */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_default()"); ret = LZ4_compress_default(block, compressedBuffer, blockSize, (int)compressedBufferSize); FUZ_CHECKTEST(ret==0, "LZ4_compress_default() failed"); compressedSize = ret; /* Decompression tests */ - /* Test decoding with output size being exactly what's necessary => must work */ - FUZ_DISPLAYTEST; + /* Test decoding with output size exactly correct => must work */ + FUZ_DISPLAYTEST("LZ4_decompress_fast() with exact output buffer"); ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize); FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space"); FUZ_CHECKTEST(ret!=compressedSize, "LZ4_decompress_fast failed : did not fully read compressed data"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data"); + { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data"); + } /* Test decoding with one byte missing => must fail */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("LZ4_decompress_fast() with output buffer 1-byte too short"); decodedBuffer[blockSize-1] = 0; ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small"); FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer"); /* Test decoding with one byte too much => must fail */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large"); /* Test decoding with output size exactly what's necessary => must work */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); decodedBuffer[blockSize] = 0; ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize); FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space"); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); + { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); + } // Test decoding with more than enough output size => must work - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); decodedBuffer[blockSize] = 0; decodedBuffer[blockSize+1] = 0; ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize+1); @@ -455,18 +507,19 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data"); //FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe wrote more than (unknown) target size"); // well, is that an issue ? FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ4_decompress_safe overrun specified output buffer size"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); + { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); + } // Test decoding with output size being one byte too short => must fail - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); decodedBuffer[blockSize-1] = 0; ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-1); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short"); FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe overrun specified output buffer size"); // Test decoding with output size being 10 bytes too short => must fail - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); if (blockSize>10) { decodedBuffer[blockSize-10] = 0; ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-10); @@ -475,51 +528,51 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c } // Test decoding with input size being one byte too short => must fail - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize); // Test decoding with input size being one byte too large => must fail - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); decodedBuffer[blockSize] = 0; ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize+1, blockSize); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size"); // Test partial decoding with target output size being max/2 => must work - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize/2, blockSize); FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space"); // Test partial decoding with target output size being just below max => must work - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize-3, blockSize); FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space"); /* Test Compression with limited output size */ /* Test compression with output size being exactly what's necessary (should work) */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_default() with output buffer just the right size"); ret = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize); FUZ_CHECKTEST(ret==0, "LZ4_compress_default() failed despite sufficient space"); /* Test compression with output size being exactly what's necessary and external state (should work) */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_fast_extState() with output buffer just the right size"); ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, compressedSize, 1); FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed despite sufficient space"); /* Test HC compression with output size being exactly what's necessary (should work) */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_HC() with output buffer just the right size"); ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel); FUZ_CHECKTEST(ret==0, "LZ4_compress_HC() failed despite sufficient space"); /* Test HC compression with output size being exactly what's necessary (should work) */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC() with output buffer just the right size"); ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel); FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed despite sufficient space"); /* Test compression with missing bytes into output buffer => must fail */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_default() with output buffer a bit too short"); { int missingBytes = (FUZ_rand(&randState) % 0x3F) + 1; if (missingBytes >= compressedSize) missingBytes = compressedSize-1; missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ @@ -530,7 +583,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c } /* Test HC compression with missing bytes into output buffer => must fail */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_HC() with output buffer a bit too short"); { int missingBytes = (FUZ_rand(&randState) % 0x3F) + 1; if (missingBytes >= HCcompressedSize) missingBytes = HCcompressedSize-1; missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ @@ -546,7 +599,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /*-******************/ /* Compress using dictionary */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary of size %i", dictSize); { LZ4_stream_t LZ4_stream; LZ4_resetStream(&LZ4_stream); LZ4_compress_fast_continue (&LZ4_stream, dict, compressedBuffer, dictSize, (int)compressedBufferSize, 1); /* Just to fill hash tables */ @@ -555,75 +608,76 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c } /* Decompress with dictionary as prefix */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_decompress_fast_usingDict() with dictionary as prefix"); memcpy(decodedBuffer, dict, dictSize); ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer+dictSize, blockSize, decodedBuffer, dictSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input"); - crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); - if (crcCheck!=crcOrig) { - int i=0; - while (block[i]==decodedBuffer[i]) i++; - printf("Wrong Byte at position %i/%i\n", i, blockSize); - + { U32 const crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); + if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); } - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_decompress_safe_usingDict()"); ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); - crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); + { U32 const crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); + } /* Compress using External dictionary */ - FUZ_DISPLAYTEST; - dict -= (FUZ_rand(&randState) & 0xF) + 1; /* Separation, so it is an ExtDict */ + FUZ_DISPLAYTEST("test LZ4_compress_fast_continue(), with non-contiguous dictionary"); + dict -= (FUZ_rand(&randState) & 0xF) + 1; /* create space, so now dictionary is an ExtDict */ if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; LZ4_loadDict(&LZ4dict, dict, dictSize); blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue failed"); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary but with an output buffer too short by one byte"); LZ4_loadDict(&LZ4dict, dict, dictSize); ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1); FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary loaded with LZ4_loadDict()"); + DISPLAYLEVEL(5, " compress %i bytes from buffer(%p) into dst(%p) using dict(%p) of size %i \n", blockSize, block, decodedBuffer, dict, dictSize); LZ4_loadDict(&LZ4dict, dict, dictSize); ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue should work : enough size available within output buffer"); /* Decompress with dictionary as external */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_decompress_fast_usingDict() with dictionary as extDict"); + DISPLAYLEVEL(5, " decoding %i bytes from buffer(%p) using dict(%p) of size %i \n", blockSize, decodedBuffer, dict, dictSize); decodedBuffer[blockSize] = 0; ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); + { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); + } - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); decodedBuffer[blockSize] = 0; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); + { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); + } - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); decodedBuffer[blockSize-1] = 0; ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)"); FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size"); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); decodedBuffer[blockSize-1] = 0; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)"); FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); { U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2; if ((U32)blockSize > missingBytes) { decodedBuffer[blockSize-missingBytes] = 0; @@ -633,19 +687,19 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c } } /* Compress using external dictionary stream */ - FUZ_DISPLAYTEST; { LZ4_stream_t LZ4_stream; int expectedSize; U32 expectedCrc; + FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_loadDict()"); LZ4_loadDict(&LZ4dict, dict, dictSize); expectedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1); FUZ_CHECKTEST(expectedSize<=0, "LZ4_compress_fast_continue reference compression for extDictCtx should have succeeded"); expectedCrc = XXH32(compressedBuffer, expectedSize, 0); + FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary()"); LZ4_loadDict(&LZ4dict, dict, dictSize); - LZ4_resetStream(&LZ4_stream); LZ4_attach_dictionary(&LZ4_stream, &LZ4dict); blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1); @@ -659,13 +713,13 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(blockContinueCompressedSize != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output (%d expected vs %d actual)", expectedSize, blockContinueCompressedSize); FUZ_CHECKTEST(XXH32(compressedBuffer, blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output"); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary(), but output buffer is 1 byte too short"); LZ4_resetStream(&LZ4_stream); LZ4_attach_dictionary(&LZ4_stream, &LZ4dict); ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1); FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using extDictCtx should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); LZ4_resetStream(&LZ4_stream); LZ4_attach_dictionary(&LZ4_stream, &LZ4dict); ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1); @@ -674,7 +728,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output"); FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output"); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); LZ4_resetStream_fast(&LZ4_stream); LZ4_attach_dictionary(&LZ4_stream, &LZ4dict); ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1); @@ -685,36 +739,38 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c } /* Decompress with dictionary as external */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); decodedBuffer[blockSize] = 0; ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); + { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); + } - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); decodedBuffer[blockSize] = 0; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); + { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); + } - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); decodedBuffer[blockSize-1] = 0; ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)"); FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size"); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); decodedBuffer[blockSize-1] = 0; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)"); FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); { U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2; if ((U32)blockSize > missingBytes) { decodedBuffer[blockSize-missingBytes] = 0; @@ -724,7 +780,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c } } /* Compress HC using External dictionary */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */ if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; LZ4_resetStreamHC (&LZ4dictHC, compressionLevel); @@ -733,32 +789,32 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue failed"); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDict should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue should work : enough size available within output buffer"); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); decodedBuffer[blockSize] = 0; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - if (crcCheck!=crcOrig) - FUZ_findDiff(block, decodedBuffer); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); + { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); + } /* Compress HC continue destSize */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); { int const availableSpace = (FUZ_rand(&randState) % blockSize) + 5; int consumedSize = blockSize; - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); LZ4_resetStreamHC (&LZ4dictHC, compressionLevel); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); blockContinueCompressedSize = LZ4_compress_HC_continue_destSize(&LZ4dictHC, block, compressedBuffer, &consumedSize, availableSpace); @@ -767,15 +823,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(blockContinueCompressedSize > availableSpace, "LZ4_compress_HC_continue_destSize write overflow"); FUZ_CHECKTEST(consumedSize > blockSize, "LZ4_compress_HC_continue_destSize read overflow"); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); decodedBuffer[consumedSize] = 0; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, consumedSize, dict, dictSize); FUZ_CHECKTEST(ret!=consumedSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); FUZ_CHECKTEST(decodedBuffer[consumedSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size") { U32 const crcSrc = XXH32(block, consumedSize, 0); U32 const crcDst = XXH32(decodedBuffer, consumedSize, 0); - if (crcSrc!=crcDst) - FUZ_findDiff(block, decodedBuffer); + if (crcSrc!=crcDst) FUZ_findDiff(block, decodedBuffer); FUZ_CHECKTEST(crcSrc!=crcDst, "LZ4_decompress_safe_usingDict corrupted decoded data"); } } @@ -802,6 +857,7 @@ _exit: free(CNBuffer); free(compressedBuffer); free(decodedBuffer); + FUZ_freeLowAddr(lowAddrBuffer, labSize); free(stateLZ4); free(stateLZ4HC); return result; @@ -1171,13 +1227,13 @@ int main(int argc, const char** argv) return FUZ_usage(programName); case 'v': /* verbose mode */ - argument++; g_displayLevel++; + argument++; break; case 'p': /* pause at the end */ - argument++; use_pause=1; + argument++; break; case 'i': |