From 024216ef7394b6411eeaa5b52d0cec9953a44249 Mon Sep 17 00:00:00 2001 From: Chongyu Zhu Date: Wed, 24 Apr 2019 03:13:19 +0800 Subject: lib/Makefile: Fix detection of `Darwin`. --- lib/Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index 330642a..8f21d3d 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -55,10 +55,11 @@ FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) SRCFILES := $(sort $(wildcard *.c)) +include ../Makefile.inc # OS X linker doesn't support -soname, and use different extension # see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html -ifeq ($(OS), Darwin) +ifeq ($(TARGET_OS), Darwin) SHARED_EXT = dylib SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT) SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT) @@ -70,8 +71,6 @@ else SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER) endif -include ../Makefile.inc - .PHONY: default default: lib-release -- cgit v0.12 From 2b68501ece2596bb40b27ffca1d1d8b54282d2fc Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 23 Apr 2019 12:29:39 -0700 Subject: added library build test on Mac OS-X --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2065478..c684b6c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,11 @@ matrix: os: osx compiler: clang script: + - make # test library build + - make clean - make -C tests test-lz4 MOREFLAGS='-Werror -Wconversion -Wno-sign-conversion' | tee # test scenario where `stdout` is not the console - - CFLAGS=-m32 make -C tests clean test-lz4-contentSize + - make clean + - CFLAGS=-m32 make -C tests test-lz4-contentSize # Container-based 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - name: (Precise) benchmark test -- cgit v0.12 From 7937e862554ae4cc8722dbf66f53c3da2259e0c5 Mon Sep 17 00:00:00 2001 From: "Dmitry V. Levin" Date: Tue, 23 Apr 2019 21:18:11 +0000 Subject: test-amalgamation: fix the list of prerequisites Add $(LZ4DIR)/lz4frame.c to the list of prerequisites as the rule uses that file. Fixes: b192c86b ("[amalgamation] lz4frame.c") --- tests/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index 67514e4..bcaf603 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -157,7 +157,7 @@ test32: CFLAGS+=-m32 test32: test .PHONY: test-amalgamation -test-amalgamation: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c +test-amalgamation: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c cat $(LZ4DIR)/lz4.c > lz4_all.c cat $(LZ4DIR)/lz4hc.c >> lz4_all.c cat $(LZ4DIR)/lz4frame.c >> lz4_all.c -- cgit v0.12 From 10726d4c56a54393ff105009077124472f0a369c Mon Sep 17 00:00:00 2001 From: "Dmitry V. Levin" Date: Tue, 23 Apr 2019 21:18:11 +0000 Subject: test-amalgamation: use a single cat command Use the list of prerequisites instead of listing those files manually, this way they will never fall out of sync. Also update the amalgamation example to use a single cat command. Fixes: a7e8d394 ("[amalgamation] add test") Fixes: b192c86b ("[amalgamation] lz4frame.c") --- lib/README.md | 4 +--- tests/Makefile | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/README.md b/lib/README.md index b753195..cf1505f 100644 --- a/lib/README.md +++ b/lib/README.md @@ -74,9 +74,7 @@ The following build macro can be selected at compilation time : lz4 source code can be amalgamated into a single file. One can combine all source code into `lz4_all.c` by using following command: ``` -cat lz4.c > lz4_all.c -cat lz4hc.c >> lz4_all.c -cat lz4frame.c >> lz4_all.c +cat lz4.c lz4hc.c lz4frame.c > lz4_all.c ``` (`cat` file order is important) then compile `lz4_all.c`. All `*.h` files present in `/lib` remain necessary to compile `lz4_all.c`. diff --git a/tests/Makefile b/tests/Makefile index bcaf603..f327535 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -158,9 +158,7 @@ test32: test .PHONY: test-amalgamation test-amalgamation: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c - cat $(LZ4DIR)/lz4.c > lz4_all.c - cat $(LZ4DIR)/lz4hc.c >> lz4_all.c - cat $(LZ4DIR)/lz4frame.c >> lz4_all.c + cat $^ > lz4_all.c $(CC) -I$(LZ4DIR) -c lz4_all.c $(RM) lz4_all.c -- cgit v0.12 From 8069d2ae6ff88fbaeff1c8d5bf1e4d3eedca1d3c Mon Sep 17 00:00:00 2001 From: "Dmitry V. Levin" Date: Tue, 23 Apr 2019 21:18:11 +0000 Subject: test-amalgamation: fix compilation options Use the same compilation options to compile lz4_all.c and other object files. Fixes: a7e8d394 ("[amalgamation] add test") --- tests/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index f327535..36cb26a 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -159,7 +159,7 @@ test32: test .PHONY: test-amalgamation test-amalgamation: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c cat $^ > lz4_all.c - $(CC) -I$(LZ4DIR) -c lz4_all.c + $(CC) $(CFLAGS) $(CPPFLAGS) -c lz4_all.c $(RM) lz4_all.c test-install: lz4 lib liblz4.pc -- cgit v0.12 From 0d3f85df651cf298e0a60c646e119e3e5fd90c72 Mon Sep 17 00:00:00 2001 From: "Dmitry V. Levin" Date: Tue, 23 Apr 2019 21:18:11 +0000 Subject: test-amalgamation: split the rule Change test-amalgamation to follow each-rule-makes-a-single-target idiom. Fixes: a7e8d394 ("[amalgamation] add test") --- tests/Makefile | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index 36cb26a..8f0dfd3 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -114,7 +114,7 @@ clean: frametest$(EXT) frametest32$(EXT) \ fasttest$(EXT) roundTripTest$(EXT) \ datagen$(EXT) checkTag$(EXT) \ - frameTest$(EXT) + frameTest$(EXT) lz4_all.c @$(RM) -rf $(TESTDIR) @echo Cleaning completed @@ -156,11 +156,13 @@ test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-install test32: CFLAGS+=-m32 test32: test -.PHONY: test-amalgamation -test-amalgamation: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c - cat $^ > lz4_all.c - $(CC) $(CFLAGS) $(CPPFLAGS) -c lz4_all.c - $(RM) lz4_all.c +test-amalgamation: lz4_all.o + +lz4_all.o: lz4_all.c + $(CC) $(CFLAGS) $(CPPFLAGS) -c $^ -o $@ + +lz4_all.c: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c + cat $^ > $@ test-install: lz4 lib liblz4.pc lz4_root=.. ./test_install.sh -- cgit v0.12 From ba99eac4d0f12a152426d95fd4fe49ff2cedd566 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 24 Apr 2019 10:03:02 -0700 Subject: several minor style changes recommended by clang-tidy --- lib/lz4.c | 137 +++++++++++++++++++++++++++++++++----------------------------- lib/lz4.h | 4 +- 2 files changed, 75 insertions(+), 66 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index e614c45..69143c0 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -606,9 +606,11 @@ int LZ4_sizeofState() { return LZ4_STREAMSIZE; } extern "C" { #endif -int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize); +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize); -int LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize, const void* dict, size_t dictSize); +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, + int compressedSize, int maxOutputSize, + const void* dictStart, size_t dictSize); #if defined (__cplusplus) } @@ -811,9 +813,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, tableType=%u", inputSize, tableType); /* If init conditions are not met, we don't have to mark stream * as having dirty context, since no action was taken yet */ - if (outputDirective == fillOutput && maxOutputSize < 1) return 0; /* Impossible to store anything */ - if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */ - if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ + if (outputDirective == fillOutput && maxOutputSize < 1) { return 0; } /* Impossible to store anything */ + if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) { return 0; } /* Unsupported inputSize, too large (or negative) */ + if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) { return 0; } /* Size too large (not within 64K limit) */ if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */ assert(acceleration >= 1); @@ -909,10 +911,10 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( forwardH = LZ4_hashPosition(forwardIp, tableType); LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); - if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue; /* match outside of valid area */ + if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) { continue; } /* match outside of valid area */ assert(matchIndex < current); - if ((tableType != byU16) && (matchIndex+LZ4_DISTANCE_MAX < current)) continue; /* too far */ - if (tableType == byU16) assert((current - matchIndex) <= LZ4_DISTANCE_MAX); /* too_far presumed impossible with byU16 */ + if ((tableType != byU16) && (matchIndex+LZ4_DISTANCE_MAX < current)) { continue; } /* too far */ + if (tableType == byU16) { assert((current - matchIndex) <= LZ4_DISTANCE_MAX); } /* too_far presumed impossible with byU16 */ if (LZ4_read32(match) == LZ4_read32(ip)) { if (maybe_extMem) offset = current - matchIndex; @@ -929,9 +931,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( { unsigned const litLength = (unsigned)(ip - anchor); token = op++; if ((outputDirective == limitedOutput) && /* Check output buffer overflow */ - (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)) ) + (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)) ) { return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ - + } if ((outputDirective == fillOutput) && (unlikely(op + (litLength+240)/255 /* litlen */ + litLength /* literals */ + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit))) { op--; @@ -1306,12 +1308,12 @@ static size_t LZ4_stream_t_alignment(void) LZ4_stream_t* LZ4_initStream (void* buffer, size_t size) { DEBUGLOG(5, "LZ4_initStream"); - if (buffer == NULL) return NULL; - if (size < sizeof(LZ4_stream_t)) return NULL; + if (buffer == NULL) { return NULL; } + if (size < sizeof(LZ4_stream_t)) { return NULL; } #ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : it reports an aligment of 8-bytes, while actually aligning LZ4_stream_t on 4 bytes. */ - if (((size_t)buffer) & (LZ4_stream_t_alignment() - 1)) return NULL; /* alignment check */ + if (((size_t)buffer) & (LZ4_stream_t_alignment() - 1)) { return NULL; } /* alignment check */ #endif MEM_INIT(buffer, 0, sizeof(LZ4_stream_t)); return (LZ4_stream_t*)buffer; @@ -1381,25 +1383,25 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) return (int)dict->dictSize; } -void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream) { +void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream) { /* Calling LZ4_resetStream_fast() here makes sure that changes will not be * erased by subsequent calls to LZ4_resetStream_fast() in case stream was * marked as having dirty context, e.g. requiring full reset. */ - LZ4_resetStream_fast(working_stream); + LZ4_resetStream_fast(workingStream); - if (dictionary_stream != NULL) { + if (dictionaryStream != NULL) { /* If the current offset is zero, we will never look in the * external dictionary context, since there is no value a table * entry can take that indicate a miss. In that case, we need * to bump the offset to something non-zero. */ - if (working_stream->internal_donotuse.currentOffset == 0) { - working_stream->internal_donotuse.currentOffset = 64 KB; + if (workingStream->internal_donotuse.currentOffset == 0) { + workingStream->internal_donotuse.currentOffset = 64 KB; } - working_stream->internal_donotuse.dictCtx = &(dictionary_stream->internal_donotuse); + workingStream->internal_donotuse.dictCtx = &(dictionaryStream->internal_donotuse); } else { - working_stream->internal_donotuse.dictCtx = NULL; + workingStream->internal_donotuse.dictCtx = NULL; } } @@ -1435,7 +1437,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i)", inputSize); - if (streamPtr->dirty) return 0; /* Uninitialized structure detected */ + if (streamPtr->dirty) { return 0; } /* Uninitialized structure detected */ LZ4_renormDictT(streamPtr, inputSize); /* avoid index overflow */ if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; @@ -1532,8 +1534,8 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; - if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ - if ((U32)dictSize > dict->dictSize) dictSize = (int)dict->dictSize; + if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */ + if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; } memmove(safeBuffer, previousDictEnd - dictSize, dictSize); @@ -1607,7 +1609,7 @@ LZ4_decompress_generic( const size_t dictSize /* note : = 0 if noDict */ ) { - if (src == NULL) return -1; + if (src == NULL) { return -1; } { const BYTE* ip = (const BYTE*) src; const BYTE* const iend = ip + srcSize; @@ -1636,9 +1638,9 @@ LZ4_decompress_generic( /* Special cases */ assert(lowPrefix <= op); - if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ - if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0 ? 1 : -1); - if ((endOnInput) && unlikely(srcSize==0)) return -1; + if ((endOnInput) && (unlikely(outputSize==0))) { return ((srcSize==1) && (*ip==0)) ? 0 : -1; } /* Empty output buffer */ + if ((!endOnInput) && (unlikely(outputSize==0))) { return (*ip==0 ? 1 : -1); } + if ((endOnInput) && unlikely(srcSize==0)) { return -1; } /* Currently the fast loop shows a regression on qualcomm arm chips. */ #if LZ4_FAST_DEC_LOOP @@ -1651,7 +1653,7 @@ LZ4_decompress_generic( while (1) { /* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */ assert(oend - op >= FASTLOOP_SAFE_DISTANCE); - if (endOnInput) assert(ip < iend); + if (endOnInput) { assert(ip < iend); } token = *ip++; length = token >> ML_BITS; /* literal length */ @@ -1661,18 +1663,18 @@ LZ4_decompress_generic( if (length == RUN_MASK) { variable_length_error error = ok; length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error); - if (error == initial_error) goto _output_error; - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */ - if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */ + if (error == initial_error) { goto _output_error; } + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ /* copy literals */ cpy = op+length; LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); if (endOnInput) { /* LZ4_decompress_safe() */ - if ((cpy>oend-32) || (ip+length>iend-32)) goto safe_literal_copy; + if ((cpy>oend-32) || (ip+length>iend-32)) { goto safe_literal_copy; } LZ4_wildCopy32(op, ip, cpy); } else { /* LZ4_decompress_fast() */ - if (cpy>oend-8) goto safe_literal_copy; + if (cpy>oend-8) { goto safe_literal_copy; } LZ4_wildCopy8(op, ip, cpy); /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time : * it doesn't know input length, and only relies on end-of-block properties */ } @@ -1682,14 +1684,14 @@ LZ4_decompress_generic( if (endOnInput) { /* LZ4_decompress_safe() */ DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length); /* We don't need to check oend, since we check it once for each loop below */ - if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) goto safe_literal_copy; + if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) { goto safe_literal_copy; } /* Literals can only be 14, but hope compilers optimize if we copy by a register size */ memcpy(op, ip, 16); } else { /* LZ4_decompress_fast() */ /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time : * it doesn't know input length, and relies on end-of-block properties */ memcpy(op, ip, 8); - if (length > 8) memcpy(op+8, ip+8, 8); + if (length > 8) { memcpy(op+8, ip+8, 8); } } ip += length; op = cpy; } @@ -1701,13 +1703,13 @@ LZ4_decompress_generic( /* get matchlength */ length = token & ML_MASK; - if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ if (length == ML_MASK) { variable_length_error error = ok; length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error); - if (error != ok) goto _output_error; - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + if (error != ok) { goto _output_error; } + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */ length += MINMATCH; if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { goto safe_match_copy; @@ -1731,9 +1733,11 @@ LZ4_decompress_generic( /* match starting within external dictionary */ if ((dict==usingExtDict) && (match < lowPrefix)) { if (unlikely(op+length > oend-LASTLITERALS)) { - if (partialDecoding) length = MIN(length, (size_t)(oend-op)); - else goto _output_error; /* doesn't respect parsing restriction */ - } + if (partialDecoding) { + length = MIN(length, (size_t)(oend-op)); /* reach end of buffer */ + } else { + goto _output_error; /* end-of-block condition violated */ + } } if (length <= (size_t)(lowPrefix-match)) { /* match fits entirely within external dictionary : just copy */ @@ -1748,7 +1752,7 @@ LZ4_decompress_generic( if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ BYTE* const endOfMatch = op + restSize; const BYTE* copyFrom = lowPrefix; - while (op < endOfMatch) *op++ = *copyFrom++; + while (op < endOfMatch) { *op++ = *copyFrom++; } } else { memcpy(op, lowPrefix, restSize); op += restSize; @@ -1821,11 +1825,11 @@ LZ4_decompress_generic( /* decode literal length */ if (length == RUN_MASK) { - variable_length_error error = ok; - length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error); - if (error == initial_error) goto _output_error; - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */ - if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */ + variable_length_error error = ok; + length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error); + if (error == initial_error) { goto _output_error; } + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ } /* copy literals */ @@ -1839,10 +1843,10 @@ LZ4_decompress_generic( { if (partialDecoding) { if (cpy > oend) { cpy = oend; assert(op<=oend); length = (size_t)(oend-op); } /* Partial decoding : stop in the middle of literal segment */ - if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ + if ((endOnInput) && (ip+length > iend)) { goto _output_error; } /* Error : read attempt beyond end of input buffer */ } else { - if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ - if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ + if ((!endOnInput) && (cpy != oend)) { goto _output_error; } /* Error : block decoding must stop exactly there */ + if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) { goto _output_error; } /* Error : input must be consumed */ } memcpy(op, ip, length); ip += length; @@ -1921,12 +1925,12 @@ LZ4_decompress_generic( const BYTE* const matchEnd = match + mlen; BYTE* const copyEnd = op + mlen; if (matchEnd > op) { /* overlap copy */ - while (op < copyEnd) *op++ = *match++; + while (op < copyEnd) { *op++ = *match++; } } else { memcpy(op, match, mlen); } op = copyEnd; - if (op==oend) break; + if (op == oend) { break; } continue; } @@ -1946,25 +1950,26 @@ LZ4_decompress_generic( if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1); - if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ + if (cpy > oend-LASTLITERALS) { goto _output_error; } /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ if (op < oCopyLimit) { LZ4_wildCopy8(op, match, oCopyLimit); match += oCopyLimit - op; op = oCopyLimit; } - while (op < cpy) *op++ = *match++; + while (op < cpy) { *op++ = *match++; } } else { memcpy(op, match, 8); - if (length > 16) LZ4_wildCopy8(op+8, match+8, cpy); + if (length > 16) { LZ4_wildCopy8(op+8, match+8, cpy); } } op = cpy; /* wildcopy correction */ } /* end of decoding */ - if (endOnInput) + if (endOnInput) { return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ - else + } else { return (int) (((const char*)ip)-src); /* Nb of input bytes read */ + } /* Overflow error detected */ _output_error: @@ -2079,7 +2084,7 @@ LZ4_streamDecode_t* LZ4_createStreamDecode(void) int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) { - if (LZ4_stream == NULL) return 0; /* support free on NULL */ + if (LZ4_stream == NULL) { return 0; } /* support free on NULL */ FREEMEM(LZ4_stream); return 0; } @@ -2214,18 +2219,22 @@ int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressed if (dictSize==0) return LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); if (dictStart+dictSize == dest) { - if (dictSize >= 64 KB - 1) + if (dictSize >= 64 KB - 1) { return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize); - return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, dictSize); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, (size_t)dictSize); } - return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, dictSize); + assert(dictSize >= 0); + return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, (size_t)dictSize); } int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) { if (dictSize==0 || dictStart+dictSize == dest) return LZ4_decompress_fast(source, dest, originalSize); - return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, dictSize); + assert(dictSize >= 0); + return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, (size_t)dictSize); } @@ -2237,9 +2246,9 @@ int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, in { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); } -int LZ4_compress(const char* source, char* dest, int inputSize) +int LZ4_compress(const char* src, char* dest, int srcSize) { - return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); + return LZ4_compress_default(src, dest, srcSize, LZ4_compressBound(srcSize)); } int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { diff --git a/lib/lz4.h b/lib/lz4.h index a9c932c..6064967 100644 --- a/lib/lz4.h +++ b/lib/lz4.h @@ -601,8 +601,8 @@ union LZ4_streamDecode_u { #endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ /* Obsolete compression functions */ -LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* source, char* dest, int sourceSize); -LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* src, char* dest, int srcSize); +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* src, char* dest, int srcSize, int maxOutputSize); LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); -- cgit v0.12 From 9e056bc032c1f1b425bbfc6034be75f595d8cffe Mon Sep 17 00:00:00 2001 From: Brenden Eng Date: Thu, 25 Apr 2019 22:37:39 -0400 Subject: Include block checksum in worst case scenario calculation of dstCapacity --- lib/lz4frame.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index a10e4af..f131d9a 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -327,6 +327,7 @@ static size_t LZ4F_compressBound_internal(size_t srcSize, { LZ4F_preferences_t prefsNull = LZ4F_INIT_PREFERENCES; prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */ + prefsNull.frameInfo.blockChecksumFlag = LZ4F_blockChecksumEnabled; /* worst case */ { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; U32 const flush = prefsPtr->autoFlush | (srcSize==0); LZ4F_blockSizeID_t const blockID = prefsPtr->frameInfo.blockSizeID; -- cgit v0.12 From dd54ea01a207a65d4819db4222066373f6170113 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Sat, 27 Apr 2019 01:41:03 +0700 Subject: contrib: Make Meson build non optional * Update ninja version to 1.9 in CI. * Update default project version in meson script. --- .travis.yml | 6 +++--- contrib/meson/meson.build | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index c684b6c..8ebcdfa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -172,13 +172,13 @@ matrix: - tests/checkTag "$TRAVIS_BRANCH" - name: (Xenial) Meson + clang build - env: ALLOW_FAILURES=true + #env: ALLOW_FAILURES=true dist: xenial language: cpp compiler: clang install: - sudo apt-get install -qq python3 tree - - curl -o ~/ninja.zip -L 'https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip' + - curl -o ~/ninja.zip -L 'https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-linux.zip' && unzip ~/ninja.zip -d ~/.local/bin - curl -o ~/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' && python3 ~/get-pip.py --user @@ -191,7 +191,7 @@ matrix: -Ddefault_library=both -Dbuild_{programs,contrib,tests,examples}=true contrib/meson build - - cd build + - pushd build - DESTDIR=./staging ninja install - tree ./staging allow_failures: diff --git a/contrib/meson/meson.build b/contrib/meson/meson.build index bf30eae..7e364eb 100644 --- a/contrib/meson/meson.build +++ b/contrib/meson/meson.build @@ -11,7 +11,9 @@ project('lz4', ['c'], license: ['BSD', 'GPLv2'], default_options : ['c_std=c99', 'buildtype=release'], - version: '1.8.3', + # Fall-back version in case of extracting version number from + # `lz4.h` failed. + version: '1.9.1', meson_version: '>=0.47.0') cc = meson.get_compiler('c') -- cgit v0.12 From cb1be75c84e9b49c82634fe9be5f968087c98df8 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Tue, 30 Apr 2019 12:25:40 +0700 Subject: meson: Rely only on extracted version in lz4.h So now instead of warning when failing to extract version number from lz4.h, we error and stop the build instead. --- .travis.yml | 4 ++-- contrib/meson/meson.build | 13 ++++--------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8ebcdfa..ee643e5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -178,9 +178,9 @@ matrix: compiler: clang install: - sudo apt-get install -qq python3 tree - - curl -o ~/ninja.zip -L 'https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-linux.zip' + - travis_retry curl -o ~/ninja.zip -L 'https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-linux.zip' && unzip ~/ninja.zip -d ~/.local/bin - - curl -o ~/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' + - travis_retry curl -o ~/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' && python3 ~/get-pip.py --user && pip3 install --user meson script: diff --git a/contrib/meson/meson.build b/contrib/meson/meson.build index 7e364eb..c28d90a 100644 --- a/contrib/meson/meson.build +++ b/contrib/meson/meson.build @@ -11,9 +11,7 @@ project('lz4', ['c'], license: ['BSD', 'GPLv2'], default_options : ['c_std=c99', 'buildtype=release'], - # Fall-back version in case of extracting version number from - # `lz4.h` failed. - version: '1.9.1', + version: 'DUMMY', meson_version: '>=0.47.0') cc = meson.get_compiler('c') @@ -40,13 +38,10 @@ lz4_h_file = join_paths(meson.current_source_dir(), '../../lib/lz4.h') GetLz4LibraryVersion_py = files('GetLz4LibraryVersion.py') r = run_command(python3, GetLz4LibraryVersion_py, lz4_h_file) if r.returncode() == 0 - output = r.stdout().strip() - if output.version_compare('>@0@'.format(lz4_version)) - lz4_version = output - message('Project version is now: @0@'.format(lz4_version)) - endif + lz4_version = r.stdout().strip() + message('Project version is now: @0@'.format(lz4_version)) else - warning('Cannot find project version in @0@'.format(lz4_h_file)) + error('Cannot find project version in @0@'.format(lz4_h_file)) endif lz4_libversion = lz4_version -- cgit v0.12 From 605d811e6cc94736dd609c644404dd24c013fd6f Mon Sep 17 00:00:00 2001 From: George Prekas Date: Fri, 3 May 2019 11:44:56 -0500 Subject: enable LZ4_FAST_DEC_LOOP build macro on aarch64/GCC by default --- lib/lz4.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/lz4.c b/lib/lz4.c index 69143c0..070dd7e 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -317,6 +317,11 @@ static const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3}; #ifndef LZ4_FAST_DEC_LOOP # if defined(__i386__) || defined(__x86_64__) # define LZ4_FAST_DEC_LOOP 1 +# elif defined(__aarch64__) && !defined(__clang__) + /* On aarch64, we disable this optimization for clang because on certain + * mobile chipsets and clang, it reduces performance. For more information + * refer to https://github.com/lz4/lz4/pull/707. */ +# define LZ4_FAST_DEC_LOOP 1 # else # define LZ4_FAST_DEC_LOOP 0 # endif -- cgit v0.12 From 98a86c8ef6ea3202f8cb52772144d0f744bd5c73 Mon Sep 17 00:00:00 2001 From: gstedman Date: Fri, 10 May 2019 16:54:05 +0100 Subject: Add multiframe report to --list command --- lib/lz4frame.c | 4 +- lib/lz4frame.h | 9 ++ programs/lz4io.c | 377 +++++++++++++++++++++++++++++++++++++------------ programs/lz4io.h | 2 + tests/Makefile | 7 +- tests/test-lz4-list.py | 282 ++++++++++++++++++++++++++++++++++++ 6 files changed, 585 insertions(+), 96 deletions(-) create mode 100644 tests/test-lz4-list.py diff --git a/lib/lz4frame.c b/lib/lz4frame.c index f131d9a..95b8b8e 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -213,8 +213,8 @@ static void LZ4F_writeLE64 (void* dst, U64 value64) static const size_t minFHSize = LZ4F_HEADER_SIZE_MIN; /* 7 */ static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 19 */ -static const size_t BHSize = 4; /* block header : size, and compress flag */ -static const size_t BFSize = 4; /* block footer : checksum (optional) */ +static const size_t BHSize = LZ4F_BLOCK_HEADER_SIZE; /* block header : size, and compress flag */ +static const size_t BFSize = LZ4F_BLOCK_CHECKSUM_SIZE; /* block footer : checksum (optional) */ /*-************************************ diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 742c252..391e484 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -253,6 +253,15 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); #define LZ4F_HEADER_SIZE_MIN 7 /* LZ4 Frame header size can vary, depending on selected paramaters */ #define LZ4F_HEADER_SIZE_MAX 19 +/* Size in bytes of a block header in little-endian format. Highest bit indicates if block data is uncompressed */ +#define LZ4F_BLOCK_HEADER_SIZE 4 + +/* Size in bytes of a block checksum footer in little-endian format. */ +#define LZ4F_BLOCK_CHECKSUM_SIZE 4 + +/* Size in bytes of the content checksum. */ +#define LZ4F_CONTENT_CHECKSUM_SIZE 4 + /*! LZ4F_compressBegin() : * will write the frame header into dstBuffer. * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. diff --git a/programs/lz4io.c b/programs/lz4io.c index 960c451..c27a0ad 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -1278,82 +1278,118 @@ int LZ4IO_decompressMultipleFilenames(LZ4IO_prefs_t* const prefs, /* ********************** LZ4 --list command *********************** */ /* ********************************************************************* */ +typedef enum +{ + lz4Frame = 0, + legacyFrame, + skippableFrame +} LZ4IO_frameType_t; + +typedef struct { + LZ4F_frameInfo_t lz4FrameInfo; + LZ4IO_frameType_t frameType; +} LZ4IO_frameInfo_t; + +#define LZ4IO_INIT_FRAMEINFO { LZ4F_INIT_FRAMEINFO, lz4Frame } + typedef struct { - LZ4F_frameInfo_t frameInfo; const char* fileName; unsigned long long fileSize; + unsigned long long frameCount; + LZ4IO_frameInfo_t frameSummary; + unsigned short eqFrameTypes; + unsigned short eqBlockTypes; + unsigned short allContentSize; } LZ4IO_cFileInfo_t; -#define LZ4IO_INIT_CFILEINFO { LZ4F_INIT_FRAMEINFO, NULL, 0ULL } - +#define LZ4IO_INIT_CFILEINFO { NULL, 0ULL, 0, LZ4IO_INIT_FRAMEINFO, 1, 1, 1 } typedef enum { LZ4IO_LZ4F_OK, LZ4IO_format_not_known, LZ4IO_not_a_file } LZ4IO_infoResult; -/* This function is limited, - * it only works fine for a file consisting of a single valid frame using LZ4 Frame specification. - * It will not look at content beyond first frame header. - * It's also unable to parse legacy frames, nor skippable ones. - * - * Things to improve : - * - check the entire file for additional content after first frame - * + combine results from multiple frames, give total - * - Optional : - * + report nb of blocks, hence max. possible decompressed size (when not reported in header) - */ -static LZ4IO_infoResult -LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filename) -{ - LZ4IO_infoResult result = LZ4IO_format_not_known; /* default result (error) */ - - if (!UTIL_isRegFile(input_filename)) return LZ4IO_not_a_file; - cfinfo->fileSize = UTIL_getFileSize(input_filename); +static const char * LZ4IO_frameTypeNames[]={"LZ4Frame", "LegacyFrame", "SkippableFrame" }; - /* Get filename without path prefix */ - { const char* b = strrchr(input_filename, '/'); - if (!b) { - b = strrchr(input_filename, '\\'); - } - if (b && b != input_filename) { - b++; - } else { - b = input_filename; +/* Read block headers and skip block data + Return total blocks size for this frame including headers, + block checksums and content checksums. + returns 0 in case it can't succesfully skip block data. + Assumes SEEK_CUR after frame header. + */ +static unsigned long long LZ4IO_skipBlocksData(FILE* finput, + const LZ4F_blockChecksum_t blockChecksumFlag, + const LZ4F_contentChecksum_t contentChecksumFlag){ + unsigned char blockInfo[LZ4F_BLOCK_HEADER_SIZE]; + unsigned long long totalBlocksSize = 0; + for(;;){ + if (!fread(blockInfo, 1, LZ4F_BLOCK_HEADER_SIZE, finput)){ + if (feof(finput)) return totalBlocksSize; + return 0; + } + totalBlocksSize += LZ4F_BLOCK_HEADER_SIZE; + { + const unsigned long nextCBlockSize = LZ4IO_readLE32(&blockInfo) & 0x7FFFFFFFU; + const unsigned long nextBlock = nextCBlockSize + (blockChecksumFlag * LZ4F_BLOCK_CHECKSUM_SIZE); + if (nextCBlockSize == 0){ + /* Reached EndMark */ + if(contentChecksumFlag){ + /* Skip content checksum */ + if(fseek(finput, LZ4F_CONTENT_CHECKSUM_SIZE, SEEK_CUR) != 0){ + return 0; + } + totalBlocksSize += LZ4F_CONTENT_CHECKSUM_SIZE; } - cfinfo->fileName = b; + break; + } + totalBlocksSize += nextBlock; + /* skip to the next block */ + if (fseek(finput, nextBlock, SEEK_CUR) != 0){ + return 0; + } } + } + return totalBlocksSize; +} - /* Read file and extract header */ - { size_t const hSize = LZ4F_HEADER_SIZE_MAX; - size_t readSize=0; - - void* const buffer = malloc(hSize); - if (!buffer) EXM_THROW(21, "Allocation error : not enough memory"); - - { FILE* const finput = LZ4IO_openSrcFile(input_filename); - if (finput) { - readSize = fread(buffer, 1, hSize, finput); - fclose(finput); - } } - - if (readSize > 0) { - LZ4F_dctx* dctx; - if (!LZ4F_isError(LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION))) { - if (!LZ4F_isError(LZ4F_getFrameInfo(dctx, &cfinfo->frameInfo, buffer, &readSize))) { - result = LZ4IO_LZ4F_OK; - } } - LZ4F_freeDecompressionContext(dctx); +/* For legacy frames only. + Read block headers and skip block data. + Return total blocks size for this frame including headers. + or 0 in case it can't succesfully skip block data. + This works as long as legacy block header size = magic number size. + Assumes SEEK_CUR after frame header. + */ +static unsigned long long LZ4IO_skipLegacyBlocksData(FILE* finput){ + unsigned char blockInfo[LZIO_LEGACY_BLOCK_HEADER_SIZE]; + unsigned long long totalBlocksSize = 0; + if(LZIO_LEGACY_BLOCK_HEADER_SIZE != MAGICNUMBER_SIZE){ + DISPLAYLEVEL(4, "Legacy block header size not equal to magic number size. Cannot skip blocks"); + return 0; + } + for(;;){ + if (!fread(blockInfo, 1, LZIO_LEGACY_BLOCK_HEADER_SIZE, finput)){ + if (feof(finput)) return totalBlocksSize; + return 0; + } + { const unsigned int nextCBlockSize = LZ4IO_readLE32(&blockInfo); + if( nextCBlockSize == LEGACY_MAGICNUMBER || + nextCBlockSize == LZ4IO_MAGICNUMBER || + LZ4IO_isSkippableMagicNumber(nextCBlockSize)){ + /* Rewind back. we want cursor at the begining of next frame.*/ + if (fseek(finput, -LZIO_LEGACY_BLOCK_HEADER_SIZE, SEEK_CUR) != 0){ + return 0; } - - /* clean */ - free(buffer); + break; + } + totalBlocksSize += LZIO_LEGACY_BLOCK_HEADER_SIZE + nextCBlockSize; + /* skip to the next block */ + if (fseek(finput, nextCBlockSize, SEEK_CUR) != 0){ + return 0; + } } - - return result; + } + return totalBlocksSize; } - /* buffer : must be a valid memory area of at least 4 bytes */ -const char* LZ4IO_blockTypeID(int sizeID, int blockMode, char* buffer) -{ +const char* LZ4IO_blockTypeID(int sizeID, int blockMode, char* buffer){ buffer[0] = 'B'; assert(sizeID >= 4); assert(sizeID <=7); buffer[1] = (char)(sizeID + '0'); @@ -1362,47 +1398,204 @@ const char* LZ4IO_blockTypeID(int sizeID, int blockMode, char* buffer) return buffer; } +/* buffer : must be valid memory area of at least 10 bytes */ +static const char* LZ4IO_toHuman(long double size, char *buf){ + const char units[] = {"\0KMGTPEZY"}; + size_t i = 0; + for(;size>=1024;i++) size /= 1024; + sprintf(buf, "%.2Lf%c", size, units[i]); + return buf; +} -int LZ4IO_displayCompressedFilesInfo(const char** inFileNames, size_t ifnIdx) +/* Get filename without path prefix */ +static const char* LZ4IO_baseName(const char* input_filename) { + const char* b = strrchr(input_filename, '/'); + if (!b) b = strrchr(input_filename, '\\'); + return b ? b + 1 : b; +} + +/* Report frame/s information in verbose mode. + * Will populate file info with fileName and contentSize where applicable. + * - TODO : + * + report nb of blocks, hence max. possible decompressed size (when not reported in header) + */ +static LZ4IO_infoResult +LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filename) { - int result = 0; - size_t idx; + LZ4IO_infoResult result = LZ4IO_format_not_known; /* default result (error) */ + unsigned char buffer[LZ4F_HEADER_SIZE_MAX]; + FILE* const finput = LZ4IO_openSrcFile(input_filename); + cfinfo->fileSize = UTIL_getFileSize(input_filename); - DISPLAY("%5s %20s %20s %10s %7s %s\n", - "Block", "Compressed", "Uncompressed", "Ratio", "Check", "Filename"); - for (idx=0; idxframeSummary.frameType != lz4Frame) cfinfo->eqFrameTypes = 0; + /* Get frame info */ + { const size_t readBytes = fread(buffer+MAGICNUMBER_SIZE, 1, LZ4F_HEADER_SIZE_MIN-MAGICNUMBER_SIZE, finput); + if (!readBytes || ferror(finput)) EXM_THROW(71, "Error reading %s", input_filename); } + { size_t hSize = LZ4F_headerSize(&buffer, LZ4F_HEADER_SIZE_MIN); + if(!LZ4F_isError(hSize)){ + if(hSize > (LZ4F_HEADER_SIZE_MIN + MAGICNUMBER_SIZE)){ + /* We've already read LZ4F_HEADER_SIZE_MIN so read any extra until hSize*/ + const size_t readBytes = fread(buffer+LZ4F_HEADER_SIZE_MIN, 1, hSize-LZ4F_HEADER_SIZE_MIN, finput); + if (!readBytes || ferror(finput)) EXM_THROW(72, "Error reading %s", input_filename); + } + /* Create decompression context */ + { LZ4F_dctx* dctx; + if (!LZ4F_isError(LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION))) { + if (!LZ4F_isError(LZ4F_getFrameInfo(dctx, &frameInfo.lz4FrameInfo, buffer, &hSize))) { + if ((cfinfo->frameSummary.lz4FrameInfo.blockSizeID != frameInfo.lz4FrameInfo.blockSizeID || + cfinfo->frameSummary.lz4FrameInfo.blockMode != frameInfo.lz4FrameInfo.blockMode) + && cfinfo->frameCount !=0) + cfinfo->eqBlockTypes = 0; + { const unsigned long long totalBlocksSize = LZ4IO_skipBlocksData(finput, + frameInfo.lz4FrameInfo.blockChecksumFlag, + frameInfo.lz4FrameInfo.contentChecksumFlag); + if(totalBlocksSize){ + char bTypeBuffer[5]; + LZ4IO_blockTypeID(frameInfo.lz4FrameInfo.blockSizeID, frameInfo.lz4FrameInfo.blockMode, bTypeBuffer); + DISPLAYLEVEL(3, " %6llu %14s %5s %8s", + cfinfo->frameCount + 1, + LZ4IO_frameTypeNames[frameInfo.frameType], + bTypeBuffer, + frameInfo.lz4FrameInfo.contentChecksumFlag ? "XXH32" : "-"); + if(frameInfo.lz4FrameInfo.contentSize){ + { double const ratio = (double)(totalBlocksSize + hSize) / frameInfo.lz4FrameInfo.contentSize * 100; + DISPLAYLEVEL(3, " %20llu %20llu %9.2f%%\n", + totalBlocksSize + hSize, + frameInfo.lz4FrameInfo.contentSize, + ratio); } + /* Now we've consumed frameInfo we can use it to store the total contentSize */ + frameInfo.lz4FrameInfo.contentSize += cfinfo->frameSummary.lz4FrameInfo.contentSize; + } + else{ + DISPLAYLEVEL(3, " %20llu %20s %9s \n", totalBlocksSize + hSize, "-", "-"); + cfinfo->allContentSize = 0; + } + result = LZ4IO_LZ4F_OK; + } } + } + } } + } } + break; + case LEGACY_MAGICNUMBER: + frameInfo.frameType = legacyFrame; + if (cfinfo->frameSummary.frameType != legacyFrame && cfinfo->frameCount !=0) cfinfo->eqFrameTypes = 0; + cfinfo->eqBlockTypes = 0; + cfinfo->allContentSize = 0; + { const unsigned long long totalBlocksSize = LZ4IO_skipLegacyBlocksData(finput); + if (totalBlocksSize){ + DISPLAYLEVEL(3, " %6llu %14s %5s %8s %20llu %20s %9s\n", + cfinfo->frameCount + 1, + LZ4IO_frameTypeNames[frameInfo.frameType], + "-", "-", + totalBlocksSize + 4, + "-", "-"); + result = LZ4IO_LZ4F_OK; + } } + break; + case LZ4IO_SKIPPABLE0: + frameInfo.frameType = skippableFrame; + if (cfinfo->frameSummary.frameType != skippableFrame && cfinfo->frameCount !=0) cfinfo->eqFrameTypes = 0; + cfinfo->eqBlockTypes = 0; + cfinfo->allContentSize = 0; + { nbReadBytes = fread(buffer, 1, 4, finput); + if (nbReadBytes != 4) + EXM_THROW(42, "Stream error : skippable size unreadable"); + } + { unsigned const size = LZ4IO_readLE32(buffer); + int const errorNb = fseek_u32(finput, size, SEEK_CUR); + if (errorNb != 0) + EXM_THROW(43, "Stream error : cannot skip skippable area"); + DISPLAYLEVEL(3, " %6llu %14s %5s %8s %20u %20s %9s\n", + cfinfo->frameCount + 1, + "SkippableFrame", + "-", "-", size + 8, "-", "-"); + + result = LZ4IO_LZ4F_OK; + } + break; + default: + { long int const position = ftell(finput); /* only works for files < 2 GB */ + DISPLAYLEVEL(3, "Stream followed by undecodable data "); + if (position != -1L) + DISPLAYLEVEL(3, "at position %i ", (int)position); + DISPLAYLEVEL(3, "\n"); + } + break; + } + if(result != LZ4IO_LZ4F_OK){ + break; + } + cfinfo->frameSummary = frameInfo; } + cfinfo->frameCount++; + } + fclose(finput); + return result; +} + + +int LZ4IO_displayCompressedFilesInfo(const char** inFileNames, size_t ifnIdx) +{ + int result = 0; + size_t idx = 0; + if(g_displayLevel < 3){ + DISPLAY("%10s %14s %5s %11s %13s %9s %s\n", + "Frames", "Type", "Block", "Compressed", "Uncompressed", "Ratio", "Filename"); + } + for (; idx 0: + self.nvinfo_list.append(NVerboseFileInfo(line)) + + def test_frames(self): + all_concat_frames = 0 + all_concat_index = None + for i, nvinfo in enumerate(self.nvinfo_list): + if "concat-all" in nvinfo.filename: + all_concat_index = i + elif "2f--content-size" in nvinfo.filename: + self.assertEqual("2", nvinfo.frames, nvinfo.line) + all_concat_frames += 2 + else: + self.assertEqual("1", nvinfo.frames, nvinfo.line) + all_concat_frames += 1 + self.assertNotEqual(None, all_concat_index, "Couldn't find concat-all file index.") + self.assertEqual(self.nvinfo_list[all_concat_index].frames, str(all_concat_frames), self.nvinfo_list[all_concat_index].line) + + def test_frame_types(self): + for nvinfo in self.nvinfo_list: + if "-lz4f-" in nvinfo.filename: + self.assertEqual(nvinfo.type, "LZ4Frame", nvinfo.line) + elif "-legc-" in nvinfo.filename: + self.assertEqual(nvinfo.type, "LegacyFrame", nvinfo.line) + elif "-skip-" in nvinfo.filename: + self.assertEqual(nvinfo.type, "SkippableFrame", nvinfo.line) + + def test_block(self): + for nvinfo in self.nvinfo_list: + # if "-leg" in nvinfo.filename or "-skip" in nvinfo.filename: + # self.assertEqual(nvinfo.block, "-", nvinfo.line) + if "--BD" in nvinfo.filename: + self.assertRegex(nvinfo.block, "^B[0-9]+D$", nvinfo.line) + elif "--BI" in nvinfo.filename: + self.assertRegex(nvinfo.block, "^B[0-9]+I$", nvinfo.line) + + def test_compressed_size(self): + for nvinfo in self.nvinfo_list: + self.assertEqual(nvinfo.compressed, to_human(nvinfo.exp_comp_size), nvinfo.line) + + def test_ratio(self): + for nvinfo in self.nvinfo_list: + if "--content-size" in nvinfo.filename: + self.assertEqual(nvinfo.ratio, "{:.2f}%".format(float(nvinfo.exp_comp_size) / float(nvinfo.exp_unc_size) * 100), nvinfo.line) + + def test_uncompressed_size(self): + for nvinfo in self.nvinfo_list: + if "--content-size" in nvinfo.filename: + self.assertEqual(nvinfo.uncompressed, to_human(nvinfo.exp_unc_size), nvinfo.line) + + +class VerboseFileInfo(object): + def __init__(self, lines): + # Parse lines + self.frame_list = [] + self.file_frame_map = [] + for i, line in enumerate(lines): + if i == 0: + self.filename = line + continue + elif i == 1: + # Skip header + continue + frame_info = dict(zip(["frame", "type", "block", "checksum", "compressed", "uncompressed", "ratio"], line.split())) + frame_info["line"] = line + self.frame_list.append(frame_info) + + +class TestVerbose(unittest.TestCase): + @classmethod + def setUpClass(self): + # Even do we're listing 2 files to test multiline working as expected. + # we're only really interested in testing the output of the concat-all file. + self.vinfo_list = [] + start = end = 0 + output = execute("{} --list -m -v {}/test_list_concat-all.lz4 {}/test_list_*M-lz4f-2f--content-size.lz4".format(LZ4, TEMP, TEMP), print_output=True) + for i, line in enumerate(output): + if line.startswith("test_list"): + if start != 0 and end != 0: + self.vinfo_list.append(VerboseFileInfo(output[start:end])) + start = i + if not line: + end = i + self.vinfo_list.append(VerboseFileInfo(output[start:end])) + # Populate file_frame_map as a reference of the expected info + concat_file_list = glob.glob("/tmp/test_list_[!concat]*.lz4") + # One of the files has 2 frames so duplicate it in this list to map each frame 1 to a single file + for i, filename in enumerate(concat_file_list): + if "2f--content-size" in filename: + concat_file_list.insert(i, filename) + break + self.cvinfo = self.vinfo_list[0] + self.cvinfo.file_frame_map = concat_file_list + self.cvinfo.compressed_size = os.path.getsize("{}/test_list_concat-all.lz4".format(TEMP)) + + def test_filename(self): + for i, vinfo in enumerate(self.vinfo_list): + self.assertRegex(vinfo.filename, "^test_list_.*({}/{})".format(i + 1, len(self.vinfo_list))) + + def test_frame_number(self): + for vinfo in self.vinfo_list: + for i, frame_info in enumerate(vinfo.frame_list): + self.assertEqual(frame_info["frame"], str(i + 1), frame_info["line"]) + + def test_frame_type(self): + for i, frame_info in enumerate(self.cvinfo.frame_list): + if "-lz4f-" in self.cvinfo.file_frame_map[i]: + self.assertEqual(self.cvinfo.frame_list[i]["type"], "LZ4Frame", self.cvinfo.frame_list[i]["line"]) + elif "-legc-" in self.cvinfo.file_frame_map[i]: + self.assertEqual(self.cvinfo.frame_list[i]["type"], "LegacyFrame", self.cvinfo.frame_list[i]["line"]) + elif "-skip-" in self.cvinfo.file_frame_map[i]: + self.assertEqual(self.cvinfo.frame_list[i]["type"], "SkippableFrame", self.cvinfo.frame_list[i]["line"]) + + def test_block(self): + for i, frame_info in enumerate(self.cvinfo.frame_list): + if "--BD" in self.cvinfo.file_frame_map[i]: + self.assertRegex(self.cvinfo.frame_list[i]["block"], "^B[0-9]+D$", self.cvinfo.frame_list[i]["line"]) + elif "--BI" in self.cvinfo.file_frame_map[i]: + self.assertEqual(self.cvinfo.frame_list[i]["block"], "^B[0-9]+I$", self.cvinfo.frame_list[i]["line"]) + + def test_checksum(self): + for i, frame_info in enumerate(self.cvinfo.frame_list): + if "-lz4f-" in self.cvinfo.file_frame_map[i] and "--no-frame-crc" not in self.cvinfo.file_frame_map[i]: + self.assertEqual(self.cvinfo.frame_list[i]["checksum"], "XXH32", self.cvinfo.frame_list[i]["line"]) + + def test_compressed(self): + total = 0 + for i, frame_info in enumerate(self.cvinfo.frame_list): + if "-2f-" not in self.cvinfo.file_frame_map[i]: + expected_size = os.path.getsize(self.cvinfo.file_frame_map[i]) + self.assertEqual(self.cvinfo.frame_list[i]["compressed"], str(expected_size), self.cvinfo.frame_list[i]["line"]) + total += int(self.cvinfo.frame_list[i]["compressed"]) + self.assertEqual(total, self.cvinfo.compressed_size, "Expected total sum ({}) to match {} filesize".format(total, self.cvinfo.filename)) + + def test_uncompressed(self): + for i, frame_info in enumerate(self.cvinfo.frame_list): + ffm = self.cvinfo.file_frame_map[i] + if "-2f-" not in ffm and "--content-size" in ffm: + expected_size_unc = int(ffm[ffm.rindex("_") + 1:ffm.index("M")]) * 1048576 + self.assertEqual(self.cvinfo.frame_list[i]["uncompressed"], str(expected_size_unc), self.cvinfo.frame_list[i]["line"]) + + def test_ratio(self): + for i, frame_info in enumerate(self.cvinfo.frame_list): + if "--content-size" in self.cvinfo.file_frame_map[i]: + self.assertEqual(self.cvinfo.frame_list[i]['ratio'], + "{:.2f}%".format(float(self.cvinfo.frame_list[i]['compressed']) / float(self.cvinfo.frame_list[i]['uncompressed']) * 100), + self.cvinfo.frame_list[i]["line"]) + + +def to_human(size): + for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y']: + if size < 1024.0: + break + size /= 1024.0 + return "{:.2f}{}".format(size, unit) + + +def log(text): + print(time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text) + + +def errout(text, err=1): + log(text) + exit(err) + + +def execute(command, print_command=True, print_output=False, print_error=True, param_shell=True): + if os.environ.get('QEMU_SYS'): + command = "{} {}".format(os.environ['QEMU_SYS'], command) + if print_command: + log("> " + command) + popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=param_shell) + stdout_lines, stderr_lines = popen.communicate() + stderr_lines = stderr_lines.decode("utf-8") + stdout_lines = stdout_lines.decode("utf-8") + if print_output: + if stdout_lines: + print(stdout_lines) + if stderr_lines: + print(stderr_lines) + if popen.returncode is not None and popen.returncode != 0: + if stderr_lines and not print_output and print_error: + print(stderr_lines) + errout("Failed to run: {}\n".format(command, stdout_lines + stderr_lines)) + return (stdout_lines + stderr_lines).splitlines() + + +def cleanup(silent=False): + for f in glob.glob("{}/test_list*".format(TEMP)): + if not silent: + log("Deleting {}".format(f)) + os.unlink(f) + + +def datagen(file_name, size): + non_sparse_size = size // 2 + sparse_size = size - non_sparse_size + with open(file_name, "wb") as f: + f.seek(sparse_size) + f.write(os.urandom(non_sparse_size)) + + +def generate_files(): + # file format ~ test_list-f.lz4 ~ + # Generate LZ4Frames + for i in SIZES: + filename = "{}/test_list_{}M".format(TEMP, i) + log("Generating {}".format(filename)) + datagen(filename, i * MIB) + for j in ["--content-size", "-BI", "-BD", "-BX", "--no-frame-crc"]: + lz4file = "{}-lz4f-1f{}.lz4".format(filename, j) + execute("{} {} {} {}".format(LZ4, j, filename, lz4file)) + # Generate skippable frames + lz4file = "{}-skip-1f.lz4".format(filename) + skipsize = i * 1024 + skipbytes = bytes([80, 42, 77, 24]) + skipsize.to_bytes(4, byteorder='little', signed=False) + with open(lz4file, 'wb') as f: + f.write(skipbytes) + f.write(os.urandom(skipsize)) + # Generate legacy frames + lz4file = "{}-legc-1f.lz4".format(filename) + execute("{} -l {} {}".format(LZ4, filename, lz4file)) + + # Concatenate --content-size files + file_list = glob.glob("{}/test_list_*-lz4f-1f--content-size.lz4".format(TEMP)) + with open("{}/test_list_{}M-lz4f-2f--content-size.lz4".format(TEMP, sum(SIZES)), 'ab') as outfile: + for fname in file_list: + with open(fname, 'rb') as infile: + outfile.write(infile.read()) + + # Concatenate all files + file_list = glob.glob("{}/test_list_*.lz4".format(TEMP)) + with open("{}/test_list_concat-all.lz4".format(TEMP), 'ab') as outfile: + for fname in file_list: + with open(fname, 'rb') as infile: + outfile.write(infile.read()) + + +if __name__ == '__main__': + cleanup() + generate_files() + unittest.main(verbosity=2, exit=False) + cleanup(silent=True) -- cgit v0.12 From 81081f9b2d6e114b06f7a3eecc8e6a20bb88e035 Mon Sep 17 00:00:00 2001 From: gstedman Date: Thu, 16 May 2019 15:12:00 +0100 Subject: List mode improvements. Improve formatting Include static assert Use UTIL_fseek to handle potential fseek limitation Be explicit when refusing to read from stdin Properly free dctx after use Include valgrind tests --- programs/lz4cli.c | 21 ++- programs/lz4io.c | 372 +++++++++++++++++++++++++------------------------ tests/Makefile | 2 + tests/test-lz4-list.py | 4 +- 4 files changed, 208 insertions(+), 191 deletions(-) diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 39ff1ea..c83b4cc 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -141,7 +141,7 @@ static int usage_advanced(const char* exeName) DISPLAY( " -BX : enable block checksum (default:disabled) \n"); DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled) \n"); DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n"); - DISPLAY( "--list : lists information about .lz4 files. Useful if compressed with --content-size flag.\n"); + DISPLAY( "--list FILE : lists information about .lz4 files (useful for files compressed with --content-size flag)\n"); DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); DISPLAY( "--favor-decSpeed: compressed files decompress faster, but are less compressed \n"); DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %i)\n", 1); @@ -654,7 +654,6 @@ int main(int argc, const char** argv) /* No output filename ==> try to select one automatically (when possible) */ while ((!output_filename) && (multiple_inputs==0)) { - if (!IS_CONSOLE(stdout)) { /* Default to stdout whenever stdout is not the console. * Note : this policy may change in the future, therefore don't rely on it ! @@ -693,7 +692,19 @@ int main(int argc, const char** argv) break; } - if (multiple_inputs==0 && mode != om_list) assert(output_filename); + if (mode == om_list){ + /* Exit if trying to read from stdin as this isn't supported in this mode */ + if(!strcmp(input_filename, stdinmark)){ + DISPLAYLEVEL(1, "refusing to read from standard input in --list mode\n"); + exit(1); + } + if(!multiple_inputs){ + inFileNames[ifnIdx++] = input_filename; + } + } + else{ + if (multiple_inputs==0) assert(output_filename); + } /* when multiple_inputs==1, output_filename may simply be useless, * however, output_filename must be !NULL for next strcmp() tests */ if (!output_filename) output_filename = "*\\dummy^!//"; @@ -723,11 +734,7 @@ int main(int argc, const char** argv) operationResult = DEFAULT_DECOMPRESSOR(prefs, input_filename, output_filename); } } else if (mode == om_list){ - if(!multiple_inputs){ - inFileNames[ifnIdx++] = input_filename; - } operationResult = LZ4IO_displayCompressedFilesInfo(inFileNames, ifnIdx); - inFileNames=NULL; } else { /* compression is default action */ if (legacy_format) { DISPLAYLEVEL(3, "! Generating LZ4 Legacy format (deprecated) ! \n"); diff --git a/programs/lz4io.c b/programs/lz4io.c index c27a0ad..d03aa6d 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -102,6 +102,7 @@ static int g_displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result } } static const clock_t refreshRate = CLOCKS_PER_SEC / 6; static clock_t g_time = 0; +#define LZ4IO_STATIC_ASSERT(c) { enum { LZ4IO_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */ /************************************** @@ -1286,112 +1287,109 @@ typedef enum } LZ4IO_frameType_t; typedef struct { - LZ4F_frameInfo_t lz4FrameInfo; - LZ4IO_frameType_t frameType; + LZ4F_frameInfo_t lz4FrameInfo; + LZ4IO_frameType_t frameType; } LZ4IO_frameInfo_t; #define LZ4IO_INIT_FRAMEINFO { LZ4F_INIT_FRAMEINFO, lz4Frame } typedef struct { - const char* fileName; - unsigned long long fileSize; - unsigned long long frameCount; - LZ4IO_frameInfo_t frameSummary; - unsigned short eqFrameTypes; - unsigned short eqBlockTypes; - unsigned short allContentSize; + const char* fileName; + unsigned long long fileSize; + unsigned long long frameCount; + LZ4IO_frameInfo_t frameSummary; + unsigned short eqFrameTypes; + unsigned short eqBlockTypes; + unsigned short allContentSize; } LZ4IO_cFileInfo_t; #define LZ4IO_INIT_CFILEINFO { NULL, 0ULL, 0, LZ4IO_INIT_FRAMEINFO, 1, 1, 1 } typedef enum { LZ4IO_LZ4F_OK, LZ4IO_format_not_known, LZ4IO_not_a_file } LZ4IO_infoResult; -static const char * LZ4IO_frameTypeNames[]={"LZ4Frame", "LegacyFrame", "SkippableFrame" }; +static const char * LZ4IO_frameTypeNames[] = {"LZ4Frame", "LegacyFrame", "SkippableFrame" }; /* Read block headers and skip block data - Return total blocks size for this frame including headers, + Return total blocks size for this frame including block headers, block checksums and content checksums. returns 0 in case it can't succesfully skip block data. Assumes SEEK_CUR after frame header. */ static unsigned long long LZ4IO_skipBlocksData(FILE* finput, - const LZ4F_blockChecksum_t blockChecksumFlag, - const LZ4F_contentChecksum_t contentChecksumFlag){ - unsigned char blockInfo[LZ4F_BLOCK_HEADER_SIZE]; - unsigned long long totalBlocksSize = 0; - for(;;){ - if (!fread(blockInfo, 1, LZ4F_BLOCK_HEADER_SIZE, finput)){ - if (feof(finput)) return totalBlocksSize; - return 0; - } - totalBlocksSize += LZ4F_BLOCK_HEADER_SIZE; - { - const unsigned long nextCBlockSize = LZ4IO_readLE32(&blockInfo) & 0x7FFFFFFFU; - const unsigned long nextBlock = nextCBlockSize + (blockChecksumFlag * LZ4F_BLOCK_CHECKSUM_SIZE); - if (nextCBlockSize == 0){ - /* Reached EndMark */ - if(contentChecksumFlag){ - /* Skip content checksum */ - if(fseek(finput, LZ4F_CONTENT_CHECKSUM_SIZE, SEEK_CUR) != 0){ + const LZ4F_blockChecksum_t blockChecksumFlag, + const LZ4F_contentChecksum_t contentChecksumFlag) { + unsigned char blockInfo[LZ4F_BLOCK_HEADER_SIZE]; + unsigned long long totalBlocksSize = 0; + for (;;) { + if (!fread(blockInfo, 1, LZ4F_BLOCK_HEADER_SIZE, finput)) { + if (feof(finput)) return totalBlocksSize; return 0; - } - totalBlocksSize += LZ4F_CONTENT_CHECKSUM_SIZE; } - break; - } - totalBlocksSize += nextBlock; - /* skip to the next block */ - if (fseek(finput, nextBlock, SEEK_CUR) != 0){ - return 0; - } + totalBlocksSize += LZ4F_BLOCK_HEADER_SIZE; + { + const unsigned long nextCBlockSize = LZ4IO_readLE32(&blockInfo) & 0x7FFFFFFFU; + const unsigned long nextBlock = nextCBlockSize + (blockChecksumFlag * LZ4F_BLOCK_CHECKSUM_SIZE); + if (nextCBlockSize == 0) { + /* Reached EndMark */ + if (contentChecksumFlag) { + /* Skip content checksum */ + if (UTIL_fseek(finput, LZ4F_CONTENT_CHECKSUM_SIZE, SEEK_CUR) != 0) { + return 0; + } + totalBlocksSize += LZ4F_CONTENT_CHECKSUM_SIZE; + } + break; + } + totalBlocksSize += nextBlock; + /* skip to the next block */ + if (UTIL_fseek(finput, nextBlock, SEEK_CUR) != 0) { + return 0; + } + } } - } - return totalBlocksSize; + return totalBlocksSize; } /* For legacy frames only. Read block headers and skip block data. - Return total blocks size for this frame including headers. + Return total blocks size for this frame including block headers. or 0 in case it can't succesfully skip block data. This works as long as legacy block header size = magic number size. Assumes SEEK_CUR after frame header. */ -static unsigned long long LZ4IO_skipLegacyBlocksData(FILE* finput){ - unsigned char blockInfo[LZIO_LEGACY_BLOCK_HEADER_SIZE]; - unsigned long long totalBlocksSize = 0; - if(LZIO_LEGACY_BLOCK_HEADER_SIZE != MAGICNUMBER_SIZE){ - DISPLAYLEVEL(4, "Legacy block header size not equal to magic number size. Cannot skip blocks"); - return 0; - } - for(;;){ - if (!fread(blockInfo, 1, LZIO_LEGACY_BLOCK_HEADER_SIZE, finput)){ - if (feof(finput)) return totalBlocksSize; - return 0; - } - { const unsigned int nextCBlockSize = LZ4IO_readLE32(&blockInfo); - if( nextCBlockSize == LEGACY_MAGICNUMBER || - nextCBlockSize == LZ4IO_MAGICNUMBER || - LZ4IO_isSkippableMagicNumber(nextCBlockSize)){ - /* Rewind back. we want cursor at the begining of next frame.*/ - if (fseek(finput, -LZIO_LEGACY_BLOCK_HEADER_SIZE, SEEK_CUR) != 0){ - return 0; +static unsigned long long LZ4IO_skipLegacyBlocksData(FILE* finput) { + unsigned char blockInfo[LZIO_LEGACY_BLOCK_HEADER_SIZE]; + unsigned long long totalBlocksSize = 0; + LZ4IO_STATIC_ASSERT(LZIO_LEGACY_BLOCK_HEADER_SIZE == MAGICNUMBER_SIZE); + for (;;) { + if (!fread(blockInfo, 1, LZIO_LEGACY_BLOCK_HEADER_SIZE, finput)) { + if (feof(finput)) return totalBlocksSize; + return 0; + } + { const unsigned int nextCBlockSize = LZ4IO_readLE32(&blockInfo); + if ( nextCBlockSize == LEGACY_MAGICNUMBER || + nextCBlockSize == LZ4IO_MAGICNUMBER || + LZ4IO_isSkippableMagicNumber(nextCBlockSize)) { + /* Rewind back. we want cursor at the begining of next frame.*/ + if (fseek(finput, -LZIO_LEGACY_BLOCK_HEADER_SIZE, SEEK_CUR) != 0) { + return 0; + } + break; + } + totalBlocksSize += LZIO_LEGACY_BLOCK_HEADER_SIZE + nextCBlockSize; + /* skip to the next block */ + if (UTIL_fseek(finput, nextCBlockSize, SEEK_CUR) != 0) { + return 0; + } } - break; - } - totalBlocksSize += LZIO_LEGACY_BLOCK_HEADER_SIZE + nextCBlockSize; - /* skip to the next block */ - if (fseek(finput, nextCBlockSize, SEEK_CUR) != 0){ - return 0; - } } - } - return totalBlocksSize; + return totalBlocksSize; } /* buffer : must be a valid memory area of at least 4 bytes */ -const char* LZ4IO_blockTypeID(int sizeID, int blockMode, char* buffer){ +const char* LZ4IO_blockTypeID(int sizeID, int blockMode, char* buffer) { buffer[0] = 'B'; - assert(sizeID >= 4); assert(sizeID <=7); + assert(sizeID >= 4); assert(sizeID <= 7); buffer[1] = (char)(sizeID + '0'); buffer[2] = (blockMode == LZ4F_blockIndependent) ? 'I' : 'D'; buffer[3] = 0; @@ -1399,23 +1397,24 @@ const char* LZ4IO_blockTypeID(int sizeID, int blockMode, char* buffer){ } /* buffer : must be valid memory area of at least 10 bytes */ -static const char* LZ4IO_toHuman(long double size, char *buf){ +static const char* LZ4IO_toHuman(long double size, char *buf) { const char units[] = {"\0KMGTPEZY"}; size_t i = 0; - for(;size>=1024;i++) size /= 1024; + for (; size >= 1024; i++) size /= 1024; sprintf(buf, "%.2Lf%c", size, units[i]); return buf; } /* Get filename without path prefix */ static const char* LZ4IO_baseName(const char* input_filename) { - const char* b = strrchr(input_filename, '/'); - if (!b) b = strrchr(input_filename, '\\'); - return b ? b + 1 : b; + const char* b = strrchr(input_filename, '/'); + if (!b) b = strrchr(input_filename, '\\'); + if (!b) return input_filename; + return b ? b + 1 : b; } /* Report frame/s information in verbose mode. - * Will populate file info with fileName and contentSize where applicable. + * Will populate file info with fileName and frameSummary where applicable. * - TODO : * + report nb of blocks, hence max. possible decompressed size (when not reported in header) */ @@ -1427,91 +1426,98 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam FILE* const finput = LZ4IO_openSrcFile(input_filename); cfinfo->fileSize = UTIL_getFileSize(input_filename); - - while(!feof(finput)){ - { LZ4IO_frameInfo_t frameInfo = LZ4IO_INIT_FRAMEINFO; + while (!feof(finput)) { + LZ4IO_frameInfo_t frameInfo = LZ4IO_INIT_FRAMEINFO; unsigned magicNumber; /* Get MagicNumber */ size_t nbReadBytes = fread(buffer, 1, MAGICNUMBER_SIZE, finput); - if (nbReadBytes==0) { break; } /* EOF */ + if (nbReadBytes == 0) { break; } /* EOF */ result = LZ4IO_format_not_known; /* default result (error) */ if (nbReadBytes != MAGICNUMBER_SIZE) - EXM_THROW(40, "Unrecognized header : Magic Number unreadable"); + EXM_THROW(40, "Unrecognized header : Magic Number unreadable"); magicNumber = LZ4IO_readLE32(buffer); /* Little Endian format */ if (LZ4IO_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4IO_SKIPPABLE0; /* fold skippable magic numbers */ - switch(magicNumber) - { + switch (magicNumber) { case LZ4IO_MAGICNUMBER: - if(cfinfo->frameSummary.frameType != lz4Frame) cfinfo->eqFrameTypes = 0; + if (cfinfo->frameSummary.frameType != lz4Frame) cfinfo->eqFrameTypes = 0; /* Get frame info */ - { const size_t readBytes = fread(buffer+MAGICNUMBER_SIZE, 1, LZ4F_HEADER_SIZE_MIN-MAGICNUMBER_SIZE, finput); - if (!readBytes || ferror(finput)) EXM_THROW(71, "Error reading %s", input_filename); } - { size_t hSize = LZ4F_headerSize(&buffer, LZ4F_HEADER_SIZE_MIN); - if(!LZ4F_isError(hSize)){ - if(hSize > (LZ4F_HEADER_SIZE_MIN + MAGICNUMBER_SIZE)){ - /* We've already read LZ4F_HEADER_SIZE_MIN so read any extra until hSize*/ - const size_t readBytes = fread(buffer+LZ4F_HEADER_SIZE_MIN, 1, hSize-LZ4F_HEADER_SIZE_MIN, finput); - if (!readBytes || ferror(finput)) EXM_THROW(72, "Error reading %s", input_filename); - } - /* Create decompression context */ - { LZ4F_dctx* dctx; - if (!LZ4F_isError(LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION))) { - if (!LZ4F_isError(LZ4F_getFrameInfo(dctx, &frameInfo.lz4FrameInfo, buffer, &hSize))) { - if ((cfinfo->frameSummary.lz4FrameInfo.blockSizeID != frameInfo.lz4FrameInfo.blockSizeID || - cfinfo->frameSummary.lz4FrameInfo.blockMode != frameInfo.lz4FrameInfo.blockMode) - && cfinfo->frameCount !=0) - cfinfo->eqBlockTypes = 0; - { const unsigned long long totalBlocksSize = LZ4IO_skipBlocksData(finput, - frameInfo.lz4FrameInfo.blockChecksumFlag, - frameInfo.lz4FrameInfo.contentChecksumFlag); - if(totalBlocksSize){ - char bTypeBuffer[5]; - LZ4IO_blockTypeID(frameInfo.lz4FrameInfo.blockSizeID, frameInfo.lz4FrameInfo.blockMode, bTypeBuffer); - DISPLAYLEVEL(3, " %6llu %14s %5s %8s", - cfinfo->frameCount + 1, - LZ4IO_frameTypeNames[frameInfo.frameType], - bTypeBuffer, - frameInfo.lz4FrameInfo.contentChecksumFlag ? "XXH32" : "-"); - if(frameInfo.lz4FrameInfo.contentSize){ - { double const ratio = (double)(totalBlocksSize + hSize) / frameInfo.lz4FrameInfo.contentSize * 100; - DISPLAYLEVEL(3, " %20llu %20llu %9.2f%%\n", - totalBlocksSize + hSize, - frameInfo.lz4FrameInfo.contentSize, - ratio); } - /* Now we've consumed frameInfo we can use it to store the total contentSize */ - frameInfo.lz4FrameInfo.contentSize += cfinfo->frameSummary.lz4FrameInfo.contentSize; - } - else{ - DISPLAYLEVEL(3, " %20llu %20s %9s \n", totalBlocksSize + hSize, "-", "-"); - cfinfo->allContentSize = 0; + { const size_t readBytes = fread(buffer + MAGICNUMBER_SIZE, 1, LZ4F_HEADER_SIZE_MIN - MAGICNUMBER_SIZE, finput); + if (!readBytes || ferror(finput)) EXM_THROW(71, "Error reading %s", input_filename); + } + { size_t hSize = LZ4F_headerSize(&buffer, LZ4F_HEADER_SIZE_MIN); + if (!LZ4F_isError(hSize)) { + if (hSize > (LZ4F_HEADER_SIZE_MIN + MAGICNUMBER_SIZE)) { + /* We've already read LZ4F_HEADER_SIZE_MIN so read any extra until hSize*/ + const size_t readBytes = fread(buffer + LZ4F_HEADER_SIZE_MIN, 1, hSize - LZ4F_HEADER_SIZE_MIN, finput); + if (!readBytes || ferror(finput)) EXM_THROW(72, "Error reading %s", input_filename); + } + /* Create decompression context */ + { LZ4F_dctx* dctx; + unsigned isError = LZ4F_isError(LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION)); + if (!isError) { + isError = LZ4F_isError(LZ4F_getFrameInfo(dctx, &frameInfo.lz4FrameInfo, buffer, &hSize)); + LZ4F_freeDecompressionContext(dctx); + if (!isError) { + if ((cfinfo->frameSummary.lz4FrameInfo.blockSizeID != frameInfo.lz4FrameInfo.blockSizeID || + cfinfo->frameSummary.lz4FrameInfo.blockMode != frameInfo.lz4FrameInfo.blockMode) + && cfinfo->frameCount != 0) + cfinfo->eqBlockTypes = 0; + { const unsigned long long totalBlocksSize = LZ4IO_skipBlocksData(finput, + frameInfo.lz4FrameInfo.blockChecksumFlag, + frameInfo.lz4FrameInfo.contentChecksumFlag); + if (totalBlocksSize) { + char bTypeBuffer[5]; + LZ4IO_blockTypeID(frameInfo.lz4FrameInfo.blockSizeID, frameInfo.lz4FrameInfo.blockMode, bTypeBuffer); + DISPLAYLEVEL(3, " %6llu %14s %5s %8s", + cfinfo->frameCount + 1, + LZ4IO_frameTypeNames[frameInfo.frameType], + bTypeBuffer, + frameInfo.lz4FrameInfo.contentChecksumFlag ? "XXH32" : "-"); + if (frameInfo.lz4FrameInfo.contentSize) { + { double const ratio = (double)(totalBlocksSize + hSize) / frameInfo.lz4FrameInfo.contentSize * 100; + DISPLAYLEVEL(3, " %20llu %20llu %9.2f%%\n", + totalBlocksSize + hSize, + frameInfo.lz4FrameInfo.contentSize, + ratio); + } + /* Now we've consumed frameInfo we can use it to store the total contentSize */ + frameInfo.lz4FrameInfo.contentSize += cfinfo->frameSummary.lz4FrameInfo.contentSize; + } + else { + DISPLAYLEVEL(3, " %20llu %20s %9s \n", totalBlocksSize + hSize, "-", "-"); + cfinfo->allContentSize = 0; + } + result = LZ4IO_LZ4F_OK; + } + } + } } - result = LZ4IO_LZ4F_OK; - } } - } - } } - } } + } + } + } break; case LEGACY_MAGICNUMBER: frameInfo.frameType = legacyFrame; - if (cfinfo->frameSummary.frameType != legacyFrame && cfinfo->frameCount !=0) cfinfo->eqFrameTypes = 0; + if (cfinfo->frameSummary.frameType != legacyFrame && cfinfo->frameCount != 0) cfinfo->eqFrameTypes = 0; cfinfo->eqBlockTypes = 0; cfinfo->allContentSize = 0; - { const unsigned long long totalBlocksSize = LZ4IO_skipLegacyBlocksData(finput); - if (totalBlocksSize){ - DISPLAYLEVEL(3, " %6llu %14s %5s %8s %20llu %20s %9s\n", - cfinfo->frameCount + 1, - LZ4IO_frameTypeNames[frameInfo.frameType], - "-", "-", - totalBlocksSize + 4, - "-", "-"); - result = LZ4IO_LZ4F_OK; - } } + { const unsigned long long totalBlocksSize = LZ4IO_skipLegacyBlocksData(finput); + if (totalBlocksSize) { + DISPLAYLEVEL(3, " %6llu %14s %5s %8s %20llu %20s %9s\n", + cfinfo->frameCount + 1, + LZ4IO_frameTypeNames[frameInfo.frameType], + "-", "-", + totalBlocksSize + 4, + "-", "-"); + result = LZ4IO_LZ4F_OK; + } + } break; case LZ4IO_SKIPPABLE0: frameInfo.frameType = skippableFrame; - if (cfinfo->frameSummary.frameType != skippableFrame && cfinfo->frameCount !=0) cfinfo->eqFrameTypes = 0; + if (cfinfo->frameSummary.frameType != skippableFrame && cfinfo->frameCount != 0) cfinfo->eqFrameTypes = 0; cfinfo->eqBlockTypes = 0; cfinfo->allContentSize = 0; { nbReadBytes = fread(buffer, 1, 4, finput); @@ -1522,10 +1528,10 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam int const errorNb = fseek_u32(finput, size, SEEK_CUR); if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area"); - DISPLAYLEVEL(3, " %6llu %14s %5s %8s %20u %20s %9s\n", - cfinfo->frameCount + 1, - "SkippableFrame", - "-", "-", size + 8, "-", "-"); + DISPLAYLEVEL(3, " %6llu %14s %5s %8s %20u %20s %9s\n", + cfinfo->frameCount + 1, + "SkippableFrame", + "-", "-", size + 8, "-", "-"); result = LZ4IO_LZ4F_OK; } @@ -1537,13 +1543,13 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam DISPLAYLEVEL(3, "at position %i ", (int)position); DISPLAYLEVEL(3, "\n"); } + break; + } + if (result != LZ4IO_LZ4F_OK) { break; } - if(result != LZ4IO_LZ4F_OK){ - break; - } - cfinfo->frameSummary = frameInfo; } - cfinfo->frameCount++; + cfinfo->frameSummary = frameInfo; + cfinfo->frameCount++; } fclose(finput); return result; @@ -1554,11 +1560,11 @@ int LZ4IO_displayCompressedFilesInfo(const char** inFileNames, size_t ifnIdx) { int result = 0; size_t idx = 0; - if(g_displayLevel < 3){ - DISPLAY("%10s %14s %5s %11s %13s %9s %s\n", - "Frames", "Type", "Block", "Compressed", "Uncompressed", "Ratio", "Filename"); + if (g_displayLevel < 3) { + DISPLAY("%10s %14s %5s %11s %13s %9s %s\n", + "Frames", "Type", "Block", "Compressed", "Uncompressed", "Ratio", "Filename"); } - for (; idx Date: Thu, 16 May 2019 16:46:16 -0700 Subject: updated tests - only play listTest with `make test`, not `make all` which is limited to build - update `clangtest`, so that it's possible to disable O3 optimization, for faster processing --- .circleci/config.yml | 4 ++-- Makefile | 11 +++++++---- tests/Makefile | 4 ++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4c08cb2..a620cd1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ version: 2 jobs: build: working_directory: ~/lz4/lz4 - parallelism: 1 + parallelism: 2 shell: /bin/bash --login # CircleCI 2.0 does not support environment variables that refer to each other the same way as 1.0 did. # If any of these refer to each other, rewrite them so that they don't or see https://circleci.com/docs/2.0/env-vars/#interpolating-environment-variables-to-set-other-environment-variables . @@ -71,7 +71,7 @@ jobs: # Test # This would typically be a build job when using workflows, possibly combined with build # This is based on your 1.0 configuration file or project settings - - run: clang -v; make clangtest && make clean + - run: CFLAGS= make clangtest && make clean - run: g++ -v; make gpptest && make clean - run: gcc -v; make c_standards && make clean - run: gcc -v; g++ -v; make ctocpptest && make clean diff --git a/Makefile b/Makefile index dd731eb..e24cec5 100644 --- a/Makefile +++ b/Makefile @@ -125,11 +125,14 @@ test: $(MAKE) -C $(TESTDIR) $@ $(MAKE) -C $(EXDIR) $@ +clangtest: CFLAGS ?= -O3 +clangtest: CFLAGS += -Werror -Wconversion -Wno-sign-conversion +clangtest: CC = clang clangtest: clean - clang -v - @CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) -C $(LZ4DIR) all CC=clang - @CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) -C $(PRGDIR) all CC=clang - @CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) -C $(TESTDIR) all CC=clang + $(CC) -v + @CFLAGS="$(CFLAGS)" $(MAKE) -C $(LZ4DIR) all CC=$(CC) + @CFLAGS="$(CFLAGS)" $(MAKE) -C $(PRGDIR) all CC=$(CC) + @CFLAGS="$(CFLAGS)" $(MAKE) -C $(TESTDIR) all CC=$(CC) clangtest-native: clean clang -v diff --git a/tests/Makefile b/tests/Makefile index dbb7bd5..a38031a 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -55,7 +55,7 @@ NB_LOOPS ?= -i1 default: all -all: fullbench fuzzer frametest roundTripTest datagen checkFrame listTest +all: fullbench fuzzer frametest roundTripTest datagen checkFrame all32: CFLAGS+=-m32 all32: all @@ -153,7 +153,7 @@ list: @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs .PHONY: test -test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-install test-amalgamation +test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-install test-amalgamation listTest .PHONY: test32 test32: CFLAGS+=-m32 -- cgit v0.12 From 2be2fe43a8460521c90edcfbbb98517933429f4d Mon Sep 17 00:00:00 2001 From: Niko Dzhus Date: Fri, 24 May 2019 18:25:06 +0300 Subject: fix temporary buffer use when input size hint is respected --- lib/lz4frame.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 95b8b8e..cc7f2d5 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -1494,7 +1494,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, /* next block is a compressed block */ dctx->tmpInTarget = nextCBlockSize + crcSize; dctx->dStage = dstage_getCBlock; - if (dstPtr==dstEnd) { + if (dstPtr==dstEnd || srcPtr==srcEnd) { nextSrcSizeHint = BHSize + nextCBlockSize + crcSize; doAnotherStage = 0; } -- cgit v0.12 From 729eef61a12d381f436da180329dba74b9892aa4 Mon Sep 17 00:00:00 2001 From: gabrielstedman Date: Sat, 25 May 2019 19:57:04 +0100 Subject: Handle file not existing case #704 --- programs/lz4cli.c | 13 ++++++++++--- tests/Makefile | 1 + 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/programs/lz4cli.c b/programs/lz4cli.c index c83b4cc..3619cd5 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -648,9 +648,16 @@ int main(int argc, const char** argv) DISPLAYLEVEL(1, "refusing to read from a console\n"); exit(1); } - /* if input==stdin and no output defined, stdout becomes default output */ - if (!strcmp(input_filename, stdinmark) && !output_filename) - output_filename = stdoutmark; + if (!strcmp(input_filename, stdinmark)) { + /* if input==stdin and no output defined, stdout becomes default output */ + if (!output_filename) output_filename = stdoutmark; + } + else{ + if (!recursive && !UTIL_isRegFile(input_filename)) { + DISPLAYLEVEL(1, "%s: is not a regular file \n", input_filename); + exit(1); + } + } /* No output filename ==> try to select one automatically (when possible) */ while ((!output_filename) && (multiple_inputs==0)) { diff --git a/tests/Makefile b/tests/Makefile index a38031a..65713ef 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -376,6 +376,7 @@ test-lz4-testmode: lz4 datagen @echo "\n ---- non-existing source ----" ! $(LZ4) file-does-not-exist ! $(LZ4) -f file-does-not-exist + ! $(LZ4) -t file-does-not-exist ! $(LZ4) -fm file1-dne file2-dne @$(RM) tmp-tlt tmp-tlt1 tmp-tlt2 tmp-tlt2.lz4 -- cgit v0.12 From 4fc6b485500ac85cd6abca04bf0fdd687ade8213 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 29 May 2019 11:19:10 -0700 Subject: added test case for in-place compression --- tests/fuzzer.c | 57 +++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index a5b5c93..26e25eb 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -389,8 +389,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Test compression destSize */ FUZ_DISPLAYTEST("test LZ4_compress_destSize()"); { int srcSize = blockSize; - int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7; - char endCheck = FUZ_rand(&randState) & 255; + int const targetSize = srcSize * (int)((FUZ_rand(&randState) & 127)+1) >> 7; + char endCheck = (char)(FUZ_rand(&randState) & 255); compressedBuffer[targetSize] = endCheck; ret = LZ4_compress_destSize(block, compressedBuffer, &srcSize, targetSize); FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_destSize() result larger than dst buffer !"); @@ -400,7 +400,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c if (targetSize>0) { /* check correctness */ U32 const crcBase = XXH32(block, (size_t)srcSize, 0); - char const canary = FUZ_rand(&randState) & 255; + char const canary = (char)(FUZ_rand(&randState) & 255); FUZ_CHECKTEST((ret==0), "LZ4_compress_destSize() compression failed"); FUZ_DISPLAYTEST(); compressedSize = ret; @@ -409,7 +409,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compress_destSize"); FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data"); FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe() overwrite dst buffer !"); - { U32 const crcDec = XXH32(decodedBuffer, srcSize, 0); + { U32 const crcDec = XXH32(decodedBuffer, (size_t)srcSize, 0); FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); } DISPLAYLEVEL(5, " OK \n"); @@ -420,8 +420,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Test compression HC destSize */ FUZ_DISPLAYTEST("test LZ4_compress_HC_destSize()"); { int srcSize = blockSize; - int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7; - char const endCheck = FUZ_rand(&randState) & 255; + int const targetSize = srcSize * (int)((FUZ_rand(&randState) & 127)+1) >> 7; + char const endCheck = (char)(FUZ_rand(&randState) & 255); void* ctx = LZ4_createHC(block); FUZ_CHECKTEST(ctx==NULL, "LZ4_createHC() allocation failed"); compressedBuffer[targetSize] = endCheck; @@ -435,7 +435,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c if (targetSize>0) { /* check correctness */ U32 const crcBase = XXH32(block, (size_t)srcSize, 0); - char const canary = FUZ_rand(&randState) & 255; + char const canary = (char)(FUZ_rand(&randState) & 255); FUZ_CHECKTEST((ret==0), "LZ4_compress_HC_destSize() compression failed"); FUZ_DISPLAYTEST(); compressedSize = ret; @@ -444,7 +444,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compressHC_destSize"); FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data"); FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe() overwrite dst buffer !"); - { U32 const crcDec = XXH32(decodedBuffer, srcSize, 0); + { U32 const crcDec = XXH32(decodedBuffer, (size_t)srcSize, 0); FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); } DISPLAYLEVEL(5, " OK \n"); @@ -524,7 +524,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(r!=blockSize, "LZ4_decompress_safe did not regenerate original data"); } FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size"); - { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); } @@ -966,7 +966,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c #define testInputSize (192 KB) -#define testCompressedSize (128 KB) +#define testCompressedSize (130 KB) #define ringBufferSize (8 KB) static void FUZ_unitTests(int compressionLevel) @@ -990,17 +990,17 @@ static void FUZ_unitTests(int compressionLevel) FUZ_AddressOverflow(); /* Test decoding with empty input */ - DISPLAYLEVEL(3, "LZ4_decompress_safe() with empty input"); + DISPLAYLEVEL(3, "LZ4_decompress_safe() with empty input \n"); LZ4_decompress_safe(testCompressed, testVerify, 0, testInputSize); /* Test decoding with a one byte input */ - DISPLAYLEVEL(3, "LZ4_decompress_safe() with one byte input"); + DISPLAYLEVEL(3, "LZ4_decompress_safe() with one byte input \n"); { char const tmp = (char)0xFF; LZ4_decompress_safe(&tmp, testVerify, 1, testInputSize); } /* Test decoding shortcut edge case */ - DISPLAYLEVEL(3, "LZ4_decompress_safe() with shortcut edge case"); + DISPLAYLEVEL(3, "LZ4_decompress_safe() with shortcut edge case \n"); { char tmp[17]; /* 14 bytes of literals, followed by a 14 byte match. * Should not read beyond the end of the buffer. @@ -1013,6 +1013,31 @@ static void FUZ_unitTests(int compressionLevel) FUZ_CHECKTEST(r >= 0, "LZ4_decompress_safe() should fail"); } } + /* in-place compression test */ + DISPLAYLEVEL(3, "in-place compression using LZ4_compress_default() :"); + { size_t const sampleSize = 65 KB; + size_t const maxCSize = LZ4_COMPRESSBOUND(sampleSize); + size_t const margin = 64 KB; + size_t const outSize = maxCSize + margin; + size_t const startIndex = outSize - sampleSize; + char* const startInput = testCompressed + startIndex; + XXH32_hash_t const crcOrig = XXH32(testInput, sampleSize, 0); + int cSize; + assert(outSize < testCompressedSize); + memcpy(startInput, testInput, sampleSize); /* copy at end of buffer */ + /* compress in-place */ + cSize = LZ4_compress_default(startInput, testCompressed, sampleSize, maxCSize); + assert(cSize != 0); /* ensure compression is successful */ + assert(maxCSize < INT_MAX); + assert(cSize <= (int)maxCSize); + /* decompress and verify */ + { int const dSize = LZ4_decompress_safe(testCompressed, testVerify, cSize, testInputSize); + assert(dSize == sampleSize); /* correct size */ + { XXH32_hash_t const crcCheck = XXH32(testVerify, (size_t)dSize, 0); + assert(crcCheck == crcOrig); + } } } + DISPLAYLEVEL(3, " OK \n"); + /* LZ4 streaming tests */ { LZ4_stream_t streamingState; U64 crcOrig; @@ -1061,17 +1086,17 @@ static void FUZ_unitTests(int compressionLevel) crcOrig = XXH64_digest(&xxhOrig); memcpy (ringBuffer + rNext, testInput + iNext, messageSize); - compressedSize = LZ4_compress_fast_continue(&streamingState, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize, 1); + compressedSize = LZ4_compress_fast_continue(&streamingState, ringBuffer + rNext, testCompressed, (int)messageSize, testCompressedSize-ringBufferSize, 1); FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_fast_continue() compression failed"); - result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, messageSize); + result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, (int)messageSize); FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe_continue() test failed"); XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize); { U64 const crcNew = XXH64_digest(&xxhNewSafe); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); } - result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, messageSize); + result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, (int)messageSize); FUZ_CHECKTEST(result!=compressedSize, "ringBuffer : LZ4_decompress_fast_continue() test failed"); XXH64_update(&xxhNewFast, testVerify + dNext, messageSize); -- cgit v0.12 From b17f578a919b7e6b078cede2d52be29dd48c8e8c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 29 May 2019 12:06:13 -0700 Subject: added comments and macros for in-place (de)compression --- doc/lz4_manual.html | 38 ++++++++++++++++++++++++++++++++++++-- lib/lz4.c | 4 ---- lib/lz4.h | 48 ++++++++++++++++++++++++++++++++++++++++++++++-- tests/fuzzer.c | 3 +-- 4 files changed, 83 insertions(+), 10 deletions(-) diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html index 3a9e0db..0530966 100644 --- a/doc/lz4_manual.html +++ b/doc/lz4_manual.html @@ -40,9 +40,9 @@ Blocks are different from Frames (doc/lz4_Frame_format.md). Frames bundle both blocks and metadata in a specified manner. - This are required for compressed data to be self-contained and portable. + Embedding metadata is required for compressed data to be self-contained and portable. Frame format is delivered through a companion API, declared in lz4frame.h. - Note that the `lz4` CLI can only manage frames. + The `lz4` CLI can only manage frames.

Version


@@ -357,6 +357,40 @@ int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
  
 


 
+

+ It's possible to have input and output sharing the same buffer,
+ for highly contrained memory environments.
+ In both cases, it requires input to lay at the end of the buffer,
+ and buffer must have some margin, hence be larger than final size.
+
+ This technique is more useful for decompression,
+ since decompressed size is typically larger,
+ and margin is mostly required to avoid stripe overflow, so it's short.
+
+ For compression though, margin must be able to cope with both
+ history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX,
+ and data expansion, which can happen when input is not compressible.
+ As a consequence, buffer size requirements are much higher than average compressed size,
+ hence memory savings are limited.
+
+ There are ways to limit this cost for compression :
+ - Reduce history size, by modifying LZ4_DISTANCE_MAX.
+   Lower values will also reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX,
+   so it's a reasonable trick when inputs are known to be small.
+ - Require the compressor to deliver a "maximum compressed size".
+   When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail,
+   in which case, the return code will be 0 (zero).
+   The caller must be ready for these cases to happen,
+   and typically design a backup scheme to send data uncompressed.
+ The combination of both techniques can significantly reduce
+ the amount of margin required for in-place compression.
+ 
+


+
+
#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize)   ( (decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN)  /**< note: presumes that compressedSize < decompressedSize */
+


+
#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize)   ( (maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN)  /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */
+


 
PRIVATE DEFINITIONS
  Do not use these definitions directly.
  They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`.
diff --git a/lib/lz4.c b/lib/lz4.c
index 070dd7e..19070c2 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -412,10 +412,6 @@ static const int LZ4_minLength = (MFLIMIT+1);
 #define MB *(1 <<20)
 #define GB *(1U<<30)
 
-#ifndef LZ4_DISTANCE_MAX   /* can be user - defined at compile time */
-#  define LZ4_DISTANCE_MAX 65535
-#endif
-
 #if (LZ4_DISTANCE_MAX > 65535)   /* max supported by LZ4 format */
 #  error "LZ4_DISTANCE_MAX is too big : must be <= 65535"
 #endif
diff --git a/lib/lz4.h b/lib/lz4.h
index 6064967..ae7f52c 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -65,9 +65,9 @@ extern "C" {
 
   Blocks are different from Frames (doc/lz4_Frame_format.md).
   Frames bundle both blocks and metadata in a specified manner.
-  This are required for compressed data to be self-contained and portable.
+  Embedding metadata is required for compressed data to be self-contained and portable.
   Frame format is delivered through a companion API, declared in lz4frame.h.
-  Note that the `lz4` CLI can only manage frames.
+  The `lz4` CLI can only manage frames.
 */
 
 /*^***************************************************************
@@ -462,8 +462,51 @@ LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const c
  */
 LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream);
 
+
+/*! In-place compression and decompression
+ *
+ * It's possible to have input and output sharing the same buffer,
+ * for highly contrained memory environments.
+ * In both cases, it requires input to lay at the end of the buffer,
+ * and buffer must have some margin, hence be larger than final size.
+ *
+ * This technique is more useful for decompression,
+ * since decompressed size is typically larger,
+ * and margin is mostly required to avoid stripe overflow, so it's short.
+ *
+ * For compression though, margin must be able to cope with both
+ * history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX,
+ * and data expansion, which can happen when input is not compressible.
+ * As a consequence, buffer size requirements are much higher than average compressed size,
+ * hence memory savings are limited.
+ *
+ * There are ways to limit this cost for compression :
+ * - Reduce history size, by modifying LZ4_DISTANCE_MAX.
+ *   Lower values will also reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX,
+ *   so it's a reasonable trick when inputs are known to be small.
+ * - Require the compressor to deliver a "maximum compressed size".
+ *   When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail,
+ *   in which case, the return code will be 0 (zero).
+ *   The caller must be ready for these cases to happen,
+ *   and typically design a backup scheme to send data uncompressed.
+ * The combination of both techniques can significantly reduce
+ * the amount of margin required for in-place compression.
+ */
+
+#define LZ4_DECOMPRESS_INPLACE_MARGIN 32
+#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize)   ( (decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN)  /**< note: presumes that compressedSize < decompressedSize */
+
+#ifndef LZ4_DISTANCE_MAX   /* history window size; can be user-defined at compile time */
+#  define LZ4_DISTANCE_MAX 65535   /* set to maximum value by default */
 #endif
 
+#define LZ4_COMPRESS_INPLACE_MARGIN (LZ4_DISTANCE_MAX + 32)
+#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize)   ( (maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN)  /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */
+
+
+#endif   /* LZ4_STATIC_LINKING_ONLY */
+
+
 
 /*-************************************************************
  *  PRIVATE DEFINITIONS
@@ -567,6 +610,7 @@ union LZ4_streamDecode_u {
 } ;   /* previously typedef'd to LZ4_streamDecode_t */
 
 
+
 /*-************************************
 *  Obsolete Functions
 **************************************/
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 26e25eb..e04caa5 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1017,8 +1017,7 @@ static void FUZ_unitTests(int compressionLevel)
     DISPLAYLEVEL(3, "in-place compression using LZ4_compress_default() :");
     {   size_t const sampleSize = 65 KB;
         size_t const maxCSize = LZ4_COMPRESSBOUND(sampleSize);
-        size_t const margin = 64 KB;
-        size_t const outSize = maxCSize + margin;
+        size_t const outSize = LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCSize);
         size_t const startIndex = outSize - sampleSize;
         char*  const startInput = testCompressed + startIndex;
         XXH32_hash_t const crcOrig = XXH32(testInput, sampleSize, 0);
-- 
cgit v0.12


From 444550defad26bcbe2e62209badbba76e803e521 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 29 May 2019 12:21:14 -0700
Subject: ensure lz4.h can be included with or without LZ4_STATIC_LINKING_ONLY
 in any order

ensure correct propagation of LZ4_DISTANCE_MAX
---
 lib/lz4.c   |  1 +
 lib/lz4.h   | 16 ++++++++++++----
 lib/lz4hc.h |  3 +++
 3 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 19070c2..a198a03 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -106,6 +106,7 @@
 #define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */
 #endif
 
+#define LZ4_STATIC_LINKING_ONLY  /* LZ4_DISTANCE_MAX */
 #include "lz4.h"
 /* see also "memory routines" below */
 
diff --git a/lib/lz4.h b/lib/lz4.h
index ae7f52c..1e24ce9 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -388,6 +388,8 @@ LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecod
  */
 LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize);
 
+#endif /* LZ4_H_2983827168210 */
+
 
 /*^*************************************
  * !!!!!!   STATIC LINKING ONLY   !!!!!!
@@ -413,14 +415,17 @@ LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int sr
  * define LZ4_PUBLISH_STATIC_FUNCTIONS when building the LZ4 library.
  ******************************************************************************/
 
+#ifdef LZ4_STATIC_LINKING_ONLY
+
+#ifndef LZ4_STATIC_3504398509
+#define LZ4_STATIC_3504398509
+
 #ifdef LZ4_PUBLISH_STATIC_FUNCTIONS
 #define LZ4LIB_STATIC_API LZ4LIB_API
 #else
 #define LZ4LIB_STATIC_API
 #endif
 
-#ifdef LZ4_STATIC_LINKING_ONLY
-
 
 /*! LZ4_compress_fast_extState_fastReset() :
  *  A variant of LZ4_compress_fast_extState().
@@ -503,11 +508,14 @@ LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const
 #define LZ4_COMPRESS_INPLACE_MARGIN (LZ4_DISTANCE_MAX + 32)
 #define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize)   ( (maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN)  /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */
 
-
+#endif   /* LZ4_STATIC_3504398509 */
 #endif   /* LZ4_STATIC_LINKING_ONLY */
 
 
 
+#ifndef LZ4_H_98237428734687
+#define LZ4_H_98237428734687
+
 /*-************************************************************
  *  PRIVATE DEFINITIONS
  **************************************************************
@@ -718,7 +726,7 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or
 LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr);
 
 
-#endif /* LZ4_H_2983827168210 */
+#endif /* LZ4_H_98237428734687 */
 
 
 #if defined (__cplusplus)
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index cdc6d89..44e35bb 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -336,6 +336,9 @@ LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionL
 #ifndef LZ4_HC_SLO_098092834
 #define LZ4_HC_SLO_098092834
 
+#define LZ4_STATIC_LINKING_ONLY   /* LZ4LIB_STATIC_API */
+#include "lz4.h"
+
 #if defined (__cplusplus)
 extern "C" {
 #endif
-- 
cgit v0.12


From c5bcb4d68b44bd276523cd354424b89efe511b76 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 29 May 2019 12:56:27 -0700
Subject: fixed minor conversion warning

---
 tests/fuzzer.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index e04caa5..8107cb8 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1031,7 +1031,7 @@ static void FUZ_unitTests(int compressionLevel)
         assert(cSize <= (int)maxCSize);
         /* decompress and verify */
         {   int const dSize = LZ4_decompress_safe(testCompressed, testVerify, cSize, testInputSize);
-            assert(dSize == sampleSize);   /* correct size */
+            assert(dSize == (int)sampleSize);   /* correct size */
             {   XXH32_hash_t const crcCheck = XXH32(testVerify, (size_t)dSize, 0);
                 assert(crcCheck == crcOrig);
     }   }   }
-- 
cgit v0.12


From 76116495bfd2096f06c923ebe865c59e42f32b07 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 29 May 2019 13:14:52 -0700
Subject: some more minor conversion warnings fixes

---
 Makefile       |  2 +-
 lib/lz4.c      |  8 ++++----
 tests/fuzzer.c | 10 +++++-----
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/Makefile b/Makefile
index e24cec5..3e55046 100644
--- a/Makefile
+++ b/Makefile
@@ -49,7 +49,7 @@ allmost: lib lz4
 .PHONY: lib lib-release liblz4.a
 lib: liblz4.a
 lib lib-release liblz4.a:
-	@$(MAKE) -C $(LZ4DIR) $@
+	@CFLAGS="$(CFLAGS)" $(MAKE) -C $(LZ4DIR) $@
 
 .PHONY: lz4 lz4-release
 lz4 : liblz4.a
diff --git a/lib/lz4.c b/lib/lz4.c
index a198a03..4451bfc 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -462,7 +462,7 @@ static unsigned LZ4_NbCommonBytes (reg_t val)
             _BitScanForward64( &r, (U64)val );
             return (int)(r>>3);
 #       elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
-            return (__builtin_ctzll((U64)val) >> 3);
+            return (unsigned)__builtin_ctzll((U64)val) >> 3;
 #       else
             static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
                                                      0, 3, 1, 3, 1, 4, 2, 7,
@@ -480,7 +480,7 @@ static unsigned LZ4_NbCommonBytes (reg_t val)
             _BitScanForward( &r, (U32)val );
             return (int)(r>>3);
 #       elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
-            return (__builtin_ctz((U32)val) >> 3);
+            return (unsigned)__builtin_ctz((U32)val) >> 3;
 #       else
             static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
                                                      3, 2, 2, 1, 3, 2, 0, 1,
@@ -496,7 +496,7 @@ static unsigned LZ4_NbCommonBytes (reg_t val)
             _BitScanReverse64( &r, val );
             return (unsigned)(r>>3);
 #       elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
-            return (__builtin_clzll((U64)val) >> 3);
+            return (unsigned)__builtin_clzll((U64)val) >> 3;
 #       else
             static const U32 by32 = sizeof(val)*4;  /* 32 on 64 bits (goal), 16 on 32 bits.
                 Just to avoid some static analyzer complaining about shift by 32 on 32-bits target.
@@ -513,7 +513,7 @@ static unsigned LZ4_NbCommonBytes (reg_t val)
             _BitScanReverse( &r, (unsigned long)val );
             return (unsigned)(r>>3);
 #       elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
-            return (__builtin_clz((U32)val) >> 3);
+            return (unsigned)__builtin_clz((U32)val) >> 3;
 #       else
             unsigned r;
             if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 8107cb8..9225e5b 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1015,10 +1015,10 @@ static void FUZ_unitTests(int compressionLevel)
 
     /* in-place compression test */
     DISPLAYLEVEL(3, "in-place compression using LZ4_compress_default() :");
-    {   size_t const sampleSize = 65 KB;
-        size_t const maxCSize = LZ4_COMPRESSBOUND(sampleSize);
-        size_t const outSize = LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCSize);
-        size_t const startIndex = outSize - sampleSize;
+    {   int const sampleSize = 65 KB;
+        int const maxCSize = LZ4_COMPRESSBOUND(sampleSize);
+        int const outSize = LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCSize);
+        int const startIndex = outSize - sampleSize;
         char*  const startInput = testCompressed + startIndex;
         XXH32_hash_t const crcOrig = XXH32(testInput, sampleSize, 0);
         int cSize;
@@ -1028,7 +1028,7 @@ static void FUZ_unitTests(int compressionLevel)
         cSize = LZ4_compress_default(startInput, testCompressed, sampleSize, maxCSize);
         assert(cSize != 0);  /* ensure compression is successful */
         assert(maxCSize < INT_MAX);
-        assert(cSize <= (int)maxCSize);
+        assert(cSize <= maxCSize);
         /* decompress and verify */
         {   int const dSize = LZ4_decompress_safe(testCompressed, testVerify, cSize, testInputSize);
             assert(dSize == (int)sampleSize);   /* correct size */
-- 
cgit v0.12


From 45eba5b0303c7a960b46fa07c3f578f5051826d8 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 29 May 2019 13:17:45 -0700
Subject: one more conversion warning

---
 tests/fuzzer.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 9225e5b..0cd651d 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1022,7 +1022,7 @@ static void FUZ_unitTests(int compressionLevel)
         char*  const startInput = testCompressed + startIndex;
         XXH32_hash_t const crcOrig = XXH32(testInput, sampleSize, 0);
         int cSize;
-        assert(outSize < testCompressedSize);
+        assert(outSize < (int)testCompressedSize);
         memcpy(startInput, testInput, sampleSize);  /* copy at end of buffer */
         /* compress in-place */
         cSize = LZ4_compress_default(startInput, testCompressed, sampleSize, maxCSize);
-- 
cgit v0.12


From b2ba857a4f295af79f94207cab1b9febec14a380 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 29 May 2019 13:33:55 -0700
Subject: fuzzer: changed internal buffer size

to ensure no overflow during unit tests
---
 tests/fuzzer.c | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 0cd651d..ba995e7 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -965,7 +965,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
 }
 
 
-#define testInputSize (192 KB)
+#define testInputSize (196 KB)
 #define testCompressedSize (130 KB)
 #define ringBufferSize (8 KB)
 
@@ -1175,12 +1175,12 @@ static void FUZ_unitTests(int compressionLevel)
         {   U64 const crc64 = XXH64(testInput + 64 KB, testCompressedSize, 0);
             LZ4_resetStreamHC_fast(&sHC, compressionLevel);
             LZ4_loadDictHC(&sHC, testInput, 64 KB);
-            result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1);
-            FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result);
-            FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
-
-            result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 64 KB);
-            FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed");
+            {   int const cSize = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1);
+                FUZ_CHECKTEST(cSize==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : @return = %i", cSize);
+                FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
+                {   int const dSize = LZ4_decompress_safe_usingDict(testCompressed, testVerify, cSize, testCompressedSize, testInput, 64 KB);
+                    FUZ_CHECKTEST(dSize!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed");
+            }   }
             {   U64 const crcNew = XXH64(testVerify, testCompressedSize, 0);
                 FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() simple dictionary decompression test : corruption");
         }   }
@@ -1189,7 +1189,8 @@ static void FUZ_unitTests(int compressionLevel)
         /* multiple HC compression test with dictionary */
         {   int result1, result2;
             int segSize = testCompressedSize / 2;
-            U64 const crc64 = XXH64(testInput + segSize, testCompressedSize, 0);
+            XXH64_hash_t const crc64 = ( (void)assert((unsigned)segSize + testCompressedSize < testInputSize) ,
+                                        XXH64(testInput + segSize, testCompressedSize, 0) );
             LZ4_resetStreamHC_fast(&sHC, compressionLevel);
             LZ4_loadDictHC(&sHC, testInput, segSize);
             result1 = LZ4_compress_HC_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1);
@@ -1203,7 +1204,7 @@ static void FUZ_unitTests(int compressionLevel)
             FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 1 failed");
             result = LZ4_decompress_safe_usingDict(testCompressed+result1, testVerify+segSize, result2, segSize, testInput, 2*segSize);
             FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 2 failed");
-            {   U64 const crcNew = XXH64(testVerify, testCompressedSize, 0);
+            {   XXH64_hash_t const crcNew = XXH64(testVerify, testCompressedSize, 0);
                 FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption");
         }   }
 
-- 
cgit v0.12


From 22adbb176afa46ebe1b799f7758381da8461bfe4 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 30 May 2019 09:45:21 -0700
Subject: add more doc on in-place (de)compression

---
 doc/lz4_manual.html | 30 +++++++++++++++++++++++++-----
 lib/lz4.h           | 30 +++++++++++++++++++++++++-----
 2 files changed, 50 insertions(+), 10 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index 0530966..091f537 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -361,21 +361,35 @@ int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
  It's possible to have input and output sharing the same buffer,
  for highly contrained memory environments.
  In both cases, it requires input to lay at the end of the buffer,
- and buffer must have some margin, hence be larger than final size.
+ and decompression to start at beginning of the buffer.
+ Buffer size must feature some margin, hence be larger than final size.
+
+ |<------------------------buffer----------------------------------->|
+                             |<------------compressed data---------->|
+ |<-----------decompressed size------------------>|
+                                                  |<-----margin----->|
 
  This technique is more useful for decompression,
  since decompressed size is typically larger,
  and margin is mostly required to avoid stripe overflow, so it's short.
 
- For compression though, margin must be able to cope with both
+ In-place decompression will work inside any buffer
+ which size is >= LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize).
+ This presumes that decompressedSize > compressedSize.
+ Otherwise, it means compression actually expanded data,
+ which can happen when data is not compressible (already compressed, or encrypted),
+ and it would be more efficient to store such data with a flag indicating it's not compressed.
+
+ For compression, margin is larger, as it must be able to cope with both
  history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX,
  and data expansion, which can happen when input is not compressible.
- As a consequence, buffer size requirements are much higher than average compressed size,
- hence memory savings are limited.
+ As a consequence, buffer size requirements are much higher,
+ and memory savings offered by in-place compression are more limited.
 
  There are ways to limit this cost for compression :
  - Reduce history size, by modifying LZ4_DISTANCE_MAX.
-   Lower values will also reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX,
+   Note that it is a compile-time constant, so all future compression will apply this parameter.
+   Lower values will reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX,
    so it's a reasonable trick when inputs are known to be small.
  - Require the compressor to deliver a "maximum compressed size".
    When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail,
@@ -384,6 +398,12 @@ int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
    and typically design a backup scheme to send data uncompressed.
  The combination of both techniques can significantly reduce
  the amount of margin required for in-place compression.
+
+ In-place compression can work in any buffer
+ which size is >= LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize)
+ with maxCompressedSize == LZ4_COMPRESSBOUND(srcSize) for guaranteed compression success.
+ This macro depends on both maxCompressedSize and LZ4_DISTANCE_MAX,
+ so there are ways to reduce memory requirements by playing with them.
  
 


 
diff --git a/lib/lz4.h b/lib/lz4.h
index 1e24ce9..91dcf64 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -473,21 +473,35 @@ LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const
  * It's possible to have input and output sharing the same buffer,
  * for highly contrained memory environments.
  * In both cases, it requires input to lay at the end of the buffer,
- * and buffer must have some margin, hence be larger than final size.
+ * and decompression to start at beginning of the buffer.
+ * Buffer size must feature some margin, hence be larger than final size.
+ *
+ * |<------------------------buffer----------------------------------->|
+ *                             |<------------compressed data---------->|
+ * |<-----------decompressed size------------------>|
+ *                                                  |<-----margin----->|
  *
  * This technique is more useful for decompression,
  * since decompressed size is typically larger,
  * and margin is mostly required to avoid stripe overflow, so it's short.
  *
- * For compression though, margin must be able to cope with both
+ * In-place decompression will work inside any buffer
+ * which size is >= LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize).
+ * This presumes that decompressedSize > compressedSize.
+ * Otherwise, it means compression actually expanded data,
+ * which can happen when data is not compressible (already compressed, or encrypted),
+ * and it would be more efficient to store such data with a flag indicating it's not compressed.
+ *
+ * For compression, margin is larger, as it must be able to cope with both
  * history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX,
  * and data expansion, which can happen when input is not compressible.
- * As a consequence, buffer size requirements are much higher than average compressed size,
- * hence memory savings are limited.
+ * As a consequence, buffer size requirements are much higher,
+ * and memory savings offered by in-place compression are more limited.
  *
  * There are ways to limit this cost for compression :
  * - Reduce history size, by modifying LZ4_DISTANCE_MAX.
- *   Lower values will also reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX,
+ *   Note that it is a compile-time constant, so all future compression will apply this parameter.
+ *   Lower values will reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX,
  *   so it's a reasonable trick when inputs are known to be small.
  * - Require the compressor to deliver a "maximum compressed size".
  *   When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail,
@@ -496,6 +510,12 @@ LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const
  *   and typically design a backup scheme to send data uncompressed.
  * The combination of both techniques can significantly reduce
  * the amount of margin required for in-place compression.
+ *
+ * In-place compression can work in any buffer
+ * which size is >= LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize)
+ * with maxCompressedSize == LZ4_COMPRESSBOUND(srcSize) for guaranteed compression success.
+ * This macro depends on both maxCompressedSize and LZ4_DISTANCE_MAX,
+ * so there are ways to reduce memory requirements by playing with them.
  */
 
 #define LZ4_DECOMPRESS_INPLACE_MARGIN 32
-- 
cgit v0.12


From 6c69ae6bd69ee66b9e5051b4f88d533999240553 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 30 May 2019 16:17:47 -0700
Subject: added test case for in-place decompression

worst case, designed to make the decoder overwrite into input
---
 tests/fuzzer.c | 95 ++++++++++++++++++++++++++++++++++++++--------------------
 1 file changed, 63 insertions(+), 32 deletions(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index ba995e7..b45620b 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -207,7 +207,7 @@ static int FUZ_AddressOverflow(void)
         }
 
         {   size_t const sizeToGenerateOverflow = (size_t)(- ((uintptr_t)buffers[nbBuff-1]) + 512);
-            unsigned const nbOf255 = (unsigned)((sizeToGenerateOverflow / 255) + 1);
+            int const nbOf255 = (int)((sizeToGenerateOverflow / 255) + 1);
             char* const input = buffers[nbBuff-1];
             char* output = buffers[nbBuff];
             int r;
@@ -215,7 +215,7 @@ static int FUZ_AddressOverflow(void)
             input[1] = (char)0xFF;
             input[2] = (char)0xFF;
             input[3] = (char)0xFF;
-            { unsigned u; for(u = 4; u <= nbOf255+4; u++) input[u] = (char)0xff; }
+            { int u; for(u = 4; u <= nbOf255+4; u++) input[u] = (char)0xff; }
             r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
             if (r>0) { DISPLAY("LZ4_decompress_safe = %i \n", r); goto _overflowError; }
             input[0] = (char)0x1F;   /* Match length overflow */
@@ -366,7 +366,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         U32 testNb = 0;
         U32 randState = FUZ_rand(&coreRandState) ^ PRIME3;
         int const blockSize  = (FUZ_rand(&randState) % (FUZ_MAX_BLOCK_SIZE-1)) + 1;
-        int const blockStart = (FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize - 1)) + 1;
+        int const blockStart = (int)(FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize - 1)) + 1;
         int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
         int const dictSize = MIN(dictSizeRand, blockStart - 1);
         int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1);
@@ -578,7 +578,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
 
         /* Test partial decoding => must work */
         FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial");
-        {   size_t const missingBytes = FUZ_rand(&randState) % blockSize;
+        {   size_t const missingBytes = FUZ_rand(&randState) % (unsigned)blockSize;
             int const targetSize = (int)((size_t)blockSize - missingBytes);
             char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A;
             int const decResult = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, targetSize, blockSize);
@@ -705,7 +705,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
         FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
         FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
-        {   U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+        {   U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0);
             FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
         }
 
@@ -792,12 +792,11 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
         FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
         FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
-        {   U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+        {   U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0);
             if (crcCheck!=crcOrig) {
                 FUZ_findDiff(block, decodedBuffer);
                 EXIT_MSG("LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
-            }
-        }
+        }   }
 
         FUZ_DISPLAYTEST();
         decodedBuffer[blockSize] = 0;
@@ -1018,8 +1017,8 @@ static void FUZ_unitTests(int compressionLevel)
     {   int const sampleSize = 65 KB;
         int const maxCSize = LZ4_COMPRESSBOUND(sampleSize);
         int const outSize = LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCSize);
-        int const startIndex = outSize - sampleSize;
-        char*  const startInput = testCompressed + startIndex;
+        int const startInputIndex = outSize - sampleSize;
+        char* const startInput = testCompressed + startInputIndex;
         XXH32_hash_t const crcOrig = XXH32(testInput, sampleSize, 0);
         int cSize;
         assert(outSize < (int)testCompressedSize);
@@ -1031,12 +1030,36 @@ static void FUZ_unitTests(int compressionLevel)
         assert(cSize <= maxCSize);
         /* decompress and verify */
         {   int const dSize = LZ4_decompress_safe(testCompressed, testVerify, cSize, testInputSize);
-            assert(dSize == (int)sampleSize);   /* correct size */
+            assert(dSize == sampleSize);   /* correct size */
             {   XXH32_hash_t const crcCheck = XXH32(testVerify, (size_t)dSize, 0);
                 assert(crcCheck == crcOrig);
     }   }   }
     DISPLAYLEVEL(3, " OK \n");
 
+    /* in-place decompression test */
+    DISPLAYLEVEL(3, "in-place decompression, limit case:");
+    {   int const sampleSize = 65 KB;
+
+        FUZ_fillCompressibleNoiseBuffer(testInput, sampleSize, 0.0, &randState);
+        memset(testInput, 0, 267);   /* calculated exactly so that compressedSize == originalSize-1 */
+
+        {   XXH64_hash_t const crcOrig = XXH64(testInput, sampleSize, 0);
+            int const cSize = LZ4_compress_default(testInput, testCompressed, sampleSize, testCompressedSize);
+            assert(cSize == sampleSize - 1);  /* worst case for in-place decompression */
+
+            {   int const bufferSize = LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(sampleSize);
+                int const startInputIndex = bufferSize - cSize;
+                char* const startInput = testVerify + startInputIndex;
+                memcpy(startInput, testCompressed, cSize);
+
+                /* decompress and verify */
+                {   int const dSize = LZ4_decompress_safe(startInput, testVerify, cSize, sampleSize);
+                    assert(dSize == sampleSize);   /* correct size */
+                    {   XXH64_hash_t const crcCheck = XXH64(testVerify, (size_t)dSize, 0);
+                        assert(crcCheck == crcOrig);
+    }   }   }   }   }
+    DISPLAYLEVEL(3, " OK \n");
+
     /* LZ4 streaming tests */
     {   LZ4_stream_t  streamingState;
         U64 crcOrig;
@@ -1226,15 +1249,15 @@ static void FUZ_unitTests(int compressionLevel)
         {   XXH64_state_t crcOrigState;
             XXH64_state_t crcNewState;
             const char* dict = testInput + 3;
-            int dictSize = (FUZ_rand(&randState) & 8191);
+            size_t dictSize = (FUZ_rand(&randState) & 8191);
             char* dst = testVerify;
 
-            size_t segStart = (size_t)dictSize + 7;
-            int segSize = (FUZ_rand(&randState) & 8191);
+            size_t segStart = dictSize + 7;
+            size_t segSize = (FUZ_rand(&randState) & 8191);
             int segNb = 1;
 
             LZ4_resetStreamHC_fast(&sHC, compressionLevel);
-            LZ4_loadDictHC(&sHC, dict, dictSize);
+            LZ4_loadDictHC(&sHC, dict, (int)dictSize);
 
             XXH64_reset(&crcOrigState, 0);
             XXH64_reset(&crcNewState, 0);
@@ -1242,29 +1265,28 @@ static void FUZ_unitTests(int compressionLevel)
             while (segStart + segSize < testInputSize) {
                 XXH64_update(&crcOrigState, testInput + segStart, segSize);
                 crcOrig = XXH64_digest(&crcOrigState);
-                result = LZ4_compress_HC_continue(&sHC, testInput + segStart, testCompressed, segSize, LZ4_compressBound(segSize));
+                assert(segSize <= INT_MAX);
+                result = LZ4_compress_HC_continue(&sHC, testInput + segStart, testCompressed, (int)segSize, LZ4_compressBound((int)segSize));
                 FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result);
                 FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
 
-                result = LZ4_decompress_safe_usingDict(testCompressed, dst, result, segSize, dict, dictSize);
-                FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %i failed", segNb);
+                result = LZ4_decompress_safe_usingDict(testCompressed, dst, result, (int)segSize, dict, (int)dictSize);
+                FUZ_CHECKTEST(result!=(int)segSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %i failed", (int)segNb);
                 XXH64_update(&crcNewState, dst, segSize);
                 {   U64 const crcNew = XXH64_digest(&crcNewState);
                     if (crcOrig != crcNew) FUZ_findDiff(dst, testInput+segStart);
                     FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() part %i corruption", segNb);
                 }
 
-                assert(segSize >= 0);
                 dict = dst;
                 dictSize = segSize;
 
-                dst += (size_t)segSize + 1;
+                dst += segSize + 1;
                 segNb ++;
 
-                segStart += (size_t)segSize + (FUZ_rand(&randState) & 0xF) + 1;
+                segStart += segSize + (FUZ_rand(&randState) & 0xF) + 1;
                 segSize = (FUZ_rand(&randState) & 8191);
-            }
-        }
+        }   }
 
         /* ring buffer test */
         {   XXH64_state_t xxhOrig;
@@ -1291,18 +1313,21 @@ static void FUZ_unitTests(int compressionLevel)
                 crcOrig = XXH64_digest(&xxhOrig);
 
                 memcpy (ringBuffer + rNext, testInput + iNext, messageSize);
-                compressedSize = LZ4_compress_HC_continue(&sHC, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
+                assert(messageSize < INT_MAX);
+                compressedSize = LZ4_compress_HC_continue(&sHC, ringBuffer + rNext, testCompressed, (int)messageSize, testCompressedSize-ringBufferSize);
                 FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed");
                 FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
 
-                result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, messageSize);
+                assert(messageSize < INT_MAX);
+                result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, (int)messageSize);
                 FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe_continue() test failed");
 
                 XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize);
                 { U64 const crcNew = XXH64_digest(&xxhNewSafe);
                   FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); }
 
-                result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, messageSize);
+                assert(messageSize < INT_MAX);
+                result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, (int)messageSize);
                 FUZ_CHECKTEST(result!=compressedSize, "ringBuffer : LZ4_decompress_fast_continue() test failed");
 
                 XXH64_update(&xxhNewFast, testVerify + dNext, messageSize);
@@ -1362,14 +1387,14 @@ static void FUZ_unitTests(int compressionLevel)
             result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, ringBufferSafe + dNext, compressedSize, messageSize);
             FUZ_CHECKTEST(result!=messageSize, "64K D.ringBuffer : LZ4_decompress_safe_continue() test failed");
 
-            XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, messageSize);
+            XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, (size_t)messageSize);
             { U64 const crcNew = XXH64_digest(&xxhNewSafe);
               FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); }
 
             result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, ringBufferFast + dNext, messageSize);
             FUZ_CHECKTEST(result!=compressedSize, "64K D.ringBuffer : LZ4_decompress_fast_continue() test failed");
 
-            XXH64_update(&xxhNewFast, ringBufferFast + dNext, messageSize);
+            XXH64_update(&xxhNewFast, ringBufferFast + dNext, (size_t)messageSize);
             { U64 const crcNew = XXH64_digest(&xxhNewFast);
               FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption"); }
 
@@ -1385,7 +1410,7 @@ static void FUZ_unitTests(int compressionLevel)
             dNext = 0;
 
             while (totalMessageSize < 9 MB) {
-                XXH64_update(&xxhOrig, testInput + iNext, messageSize);
+                XXH64_update(&xxhOrig, testInput + iNext, (size_t)messageSize);
                 crcOrig = XXH64_digest(&xxhOrig);
 
                 compressedSize = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
@@ -1400,7 +1425,7 @@ static void FUZ_unitTests(int compressionLevel)
                                                       testCompressed, ringBufferSafe + dNext,
                                                       compressedSize, dBufferSize - dNext);   /* works without knowing messageSize, under assumption that messageSize <= maxMessageSize */
                 FUZ_CHECKTEST(result!=messageSize, "D.ringBuffer : LZ4_decompress_safe_continue() test failed");
-                XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, messageSize);
+                XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, (size_t)messageSize);
                 {   U64 const crcNew = XXH64_digest(&xxhNewSafe);
                     if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, ringBufferSafe + dNext);
                     FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption during D.ringBuffer test");
@@ -1409,7 +1434,7 @@ static void FUZ_unitTests(int compressionLevel)
                 /* test LZ4_decompress_fast_continue in its own buffer ringBufferFast */
                 result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, ringBufferFast + dNext, messageSize);
                 FUZ_CHECKTEST(result!=compressedSize, "D.ringBuffer : LZ4_decompress_fast_continue() test failed");
-                XXH64_update(&xxhNewFast, ringBufferFast + dNext, messageSize);
+                XXH64_update(&xxhNewFast, ringBufferFast + dNext, (size_t)messageSize);
                 {   U64 const crcNew = XXH64_digest(&xxhNewFast);
                     if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, ringBufferFast + dNext);
                     FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption during D.ringBuffer test");
@@ -1417,7 +1442,8 @@ static void FUZ_unitTests(int compressionLevel)
 
                 /* prepare next message */
                 dNext += messageSize;
-                totalMessageSize += messageSize;
+                assert(messageSize >= 0);
+                totalMessageSize += (unsigned)messageSize;
                 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
                 iNext = (FUZ_rand(&randState) & 65535);
                 if (dNext + maxMessageSize > dBufferSize) dNext = 0;
@@ -1435,6 +1461,11 @@ static void FUZ_unitTests(int compressionLevel)
 }
 
 
+
+/* =======================================
+ * CLI
+ * ======================================= */
+
 static int FUZ_usage(const char* programName)
 {
     DISPLAY( "Usage :\n");
-- 
cgit v0.12


From 676d46df2723c89ca7d80fd71d3c2dc6e5be82ec Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 30 May 2019 16:19:30 -0700
Subject: updated LZ4_DECOMPRESS_INPLACE_MARGIN

to pass worst case scenario.
Now adds margin proportional to input size to counter local expansion.
---
 lib/lz4.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index 91dcf64..383c2db 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -518,8 +518,8 @@ LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const
  * so there are ways to reduce memory requirements by playing with them.
  */
 
-#define LZ4_DECOMPRESS_INPLACE_MARGIN 32
-#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize)   ( (decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN)  /**< note: presumes that compressedSize < decompressedSize */
+#define LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize)  (((decompressedSize) >> 8) + 32)
+#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize)   ( (decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize))  /**< note: presumes that compressedSize < decompressedSize */
 
 #ifndef LZ4_DISTANCE_MAX   /* history window size; can be user-defined at compile time */
 #  define LZ4_DISTANCE_MAX 65535   /* set to maximum value by default */
-- 
cgit v0.12


From 2f9d1736fbfe5ee92b8305a2a613ed123e03264f Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 30 May 2019 16:23:53 -0700
Subject: updated API manual

---
 doc/lz4_manual.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index 091f537..083211d 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -407,7 +407,7 @@ int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
  
 


 
-
#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize)   ( (decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN)  /**< note: presumes that compressedSize < decompressedSize */
+
#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize)   ( (decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize))  /**< note: presumes that compressedSize < decompressedSize */
 


 
#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize)   ( (maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN)  /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */
 


-- 
cgit v0.12


From 89c97d5ea682ba7e2481d610a8b7a2b057a80391 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 30 May 2019 17:29:51 -0700
Subject: fullbench: added test scenario LZ4F_decompress_followHint

This emulates a streaming scenario,
where the caller follows rigorously the srcSize hints
provided as return value of LZ4F_decompress().

This is useful to show the issue in #714,
where data is uselessly copied in a tmp buffer first.
---
 tests/fullbench.c | 64 ++++++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 51 insertions(+), 13 deletions(-)

diff --git a/tests/fullbench.c b/tests/fullbench.c
index d2af662..2488a54 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -343,11 +343,45 @@ static int local_LZ4F_decompress(const char* in, char* out, int inSize, int outS
     assert(inSize >= 0);
     assert(outSize >= 0);
     result = LZ4F_decompress(g_dCtx, out, &dstSize, in, &srcSize, NULL);
-    if (result!=0) { DISPLAY("Error decompressing frame : unfinished frame\n"); exit(8); }
-    if (srcSize != (size_t)inSize) { DISPLAY("Error decompressing frame : read size incorrect\n"); exit(9); }
+    if (result!=0) { DISPLAY("Error decompressing frame : unfinished frame \n"); exit(8); }
+    if (srcSize != (size_t)inSize) { DISPLAY("Error decompressing frame : read size incorrect \n"); exit(9); }
     return (int)dstSize;
 }
 
+static int local_LZ4F_decompress_followHint(const char* src, char* dst, int srcSize, int dstSize)
+{
+    size_t totalInSize = (size_t)srcSize;
+    size_t maxOutSize = (size_t)dstSize;
+
+    size_t inPos = 0;
+    size_t inSize = 0;
+    size_t outPos = 0;
+    size_t outRemaining = maxOutSize - outPos;
+
+    do {
+        size_t const sizeHint = LZ4F_decompress(g_dCtx, dst+outPos, &outRemaining, src+inPos, &inSize, NULL);
+        assert(!LZ4F_isError(sizeHint));
+
+        inPos += inSize;
+        inSize = sizeHint;
+
+        outPos += outRemaining;
+        outRemaining = maxOutSize - outPos;
+
+        if (!sizeHint) break;
+
+    } while(1);
+
+    /* frame completed */
+    if (inPos != totalInSize) {
+        DISPLAY("Error decompressing frame : must read (%u) full frame (%u) \n",
+                (unsigned)inPos, (unsigned)totalInSize);
+        exit(10);
+    }
+    return (int)outPos;
+
+}
+
 
 #define NB_COMPRESSION_ALGORITHMS 100
 #define NB_DECOMPRESSION_ALGORITHMS 100
@@ -446,7 +480,14 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
                 for (i=0; i g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }
+                    assert(g_chunkSize > 0);
+                    if (remaining > (size_t)g_chunkSize) {
+                        chunkP[i].origSize = g_chunkSize;
+                        remaining -= (size_t)g_chunkSize;
+                    } else {
+                        chunkP[i].origSize = (int)remaining;
+                        remaining = 0;
+                    }
                     chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;
                     chunkP[i].compressedSize = 0;
                 }
@@ -565,17 +606,14 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
 #ifndef LZ4_DLL_IMPORT
             case 8: decompressionFunction = local_LZ4_decompress_safe_forceExtDict; dName = "LZ4_decompress_safe_forceExtDict"; break;
 #endif
-            case 9: decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress";
-                {   size_t const errorCode = LZ4F_compressFrame(compressed_buff, compressedBuffSize, orig_buff, benchedSize, NULL);
-                    if (LZ4F_isError(errorCode)) {
-                        DISPLAY("Error while preparing compressed frame\n");
-                        free(orig_buff);
-                        free(compressed_buff);
-                        free(chunkP);
-                        return 1;
-                    }
+            case 10:
+            case 11:
+                if (dAlgNb == 10) { decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress"; }  /* can be skipped */
+                if (dAlgNb == 11) { decompressionFunction = local_LZ4F_decompress_followHint; dName = "LZ4F_decompress_followHint"; }  /* can be skipped */
+                {   size_t const fcsize = LZ4F_compressFrame(compressed_buff, (size_t)compressedBuffSize, orig_buff, benchedSize, NULL);
+                    assert(!LZ4F_isError(fcsize));
                     chunkP[0].origSize = (int)benchedSize;
-                    chunkP[0].compressedSize = (int)errorCode;
+                    chunkP[0].compressedSize = (int)fcsize;
                     nbChunks = 1;
                     break;
                 }
-- 
cgit v0.12


From 64b5917736396754a8f93c2913f5276361831b39 Mon Sep 17 00:00:00 2001
From: Chenxi Mao 
Date: Fri, 31 May 2019 08:36:13 +0800
Subject: FAST_DEC_LOOP: only did offset check in specific condition.

When I did FAST_DEC_LOOP performance test, I found the
offset check is much more than v1.8.3

You will see the condition check difference via lzbench with dickens test case.
v1.8.3 34959
v.1.9.x 1055885

After investigate the code, we could see the difference.
v.1.8.3 SKIP the condition check if
if condition is true in:
https://github.com/lz4/lz4/blob/v1.8.3/lib/lz4.c#L1463
AND below condition is true
https://github.com/lz4/lz4/blob/v1.8.3/lib/lz4.c#L1478\
The offset check should be invoked.

v1.9.3
The offset check code will be invoked in every loop which lead to downgrade.
So the fix would be move this check to specific condition
to avoid useless condition check.

After this change, the call number is same as v1.8.3
---
 lib/lz4.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 070dd7e..9e6999f 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1708,10 +1708,9 @@ LZ4_decompress_generic(
             /* get matchlength */
             length = token & ML_MASK;
 
-            if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */
-
             if (length == ML_MASK) {
               variable_length_error error = ok;
+              if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */
               length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error);
               if (error != ok) { goto _output_error; }
                 if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */
@@ -1735,6 +1734,7 @@ LZ4_decompress_generic(
                         continue;
             }   }   }
 
+            if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */
             /* match starting within external dictionary */
             if ((dict==usingExtDict) && (match < lowPrefix)) {
                 if (unlikely(op+length > oend-LASTLITERALS)) {
-- 
cgit v0.12


From 4eec64e4d80140400ad138b18b7286439329cb08 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 31 May 2019 11:31:48 -0700
Subject: Makefile removed CFLAGS modifier

which was removing `-O3` as a side effect
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 3e55046..e24cec5 100644
--- a/Makefile
+++ b/Makefile
@@ -49,7 +49,7 @@ allmost: lib lz4
 .PHONY: lib lib-release liblz4.a
 lib: liblz4.a
 lib lib-release liblz4.a:
-	@CFLAGS="$(CFLAGS)" $(MAKE) -C $(LZ4DIR) $@
+	@$(MAKE) -C $(LZ4DIR) $@
 
 .PHONY: lz4 lz4-release
 lz4 : liblz4.a
-- 
cgit v0.12


From 33cb8518ac385835cc17be9a770b27b40cd0e15b Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 31 May 2019 11:44:37 -0700
Subject: decompress: changed final memcpy() into memmove()

for compatibility with in-place decompression scenarios.
---
 lib/lz4.c | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 4451bfc..a1d1860 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -120,6 +120,7 @@
 #  pragma warning(disable : 4293)        /* disable: C4293: too large shift (32-bits) */
 #endif  /* _MSC_VER */
 
+#define LZ4_FORCE_INLINE static
 #ifndef LZ4_FORCE_INLINE
 #  ifdef _MSC_VER    /* Visual Studio */
 #    define LZ4_FORCE_INLINE static __forceinline
@@ -707,18 +708,19 @@ static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType
     { const U16* const hashTable = (const U16*) tableBase; return hashTable[h] + srcBase; }   /* default, to ensure a return */
 }
 
-LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p,
-                                             const void* tableBase, tableType_t tableType,
-                                             const BYTE* srcBase)
+LZ4_FORCE_INLINE const BYTE*
+LZ4_getPosition(const BYTE* p,
+                const void* tableBase, tableType_t tableType,
+                const BYTE* srcBase)
 {
     U32 const h = LZ4_hashPosition(p, tableType);
     return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
 }
 
-LZ4_FORCE_INLINE void LZ4_prepareTable(
-        LZ4_stream_t_internal* const cctx,
-        const int inputSize,
-        const tableType_t tableType) {
+LZ4_FORCE_INLINE void
+LZ4_prepareTable(LZ4_stream_t_internal* const cctx,
+           const int inputSize,
+           const tableType_t tableType) {
     /* If compression failed during the previous step, then the context
      * is marked as dirty, therefore, it has to be fully reset.
      */
@@ -733,9 +735,10 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
      * out if it's safe to leave as is or whether it needs to be reset.
      */
     if (cctx->tableType != clearedTable) {
+        assert(inputSize >= 0);
         if (cctx->tableType != tableType
-          || (tableType == byU16 && cctx->currentOffset + inputSize >= 0xFFFFU)
-          || (tableType == byU32 && cctx->currentOffset > 1 GB)
+          || ((tableType == byU16) && cctx->currentOffset + (unsigned)inputSize >= 0xFFFFU)
+          || ((tableType == byU32) && cctx->currentOffset > 1 GB)
           || tableType == byPtr
           || inputSize >= 4 KB)
         {
@@ -1850,7 +1853,7 @@ LZ4_decompress_generic(
                     if ((!endOnInput) && (cpy != oend)) { goto _output_error; }   /* Error : block decoding must stop exactly there */
                     if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) { goto _output_error; } /* Error : input must be consumed */
                 }
-                memcpy(op, ip, length);
+                memmove(op, ip, length);  /* supports overlapping memory regions, which only matters for in-place decompression scenarios */
                 ip += length;
                 op += length;
                 if (!partialDecoding || (cpy == oend)) {
-- 
cgit v0.12


From 5997e139f53169fa3a1c1b4418d2452a90b01602 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 31 May 2019 11:56:59 -0700
Subject: added more details for in-place documentation

---
 doc/lz4_manual.html | 25 +++++++++++++------------
 lib/lz4.h           | 29 +++++++++++++++--------------
 2 files changed, 28 insertions(+), 26 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index 083211d..1480089 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -364,23 +364,23 @@ int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
  and decompression to start at beginning of the buffer.
  Buffer size must feature some margin, hence be larger than final size.
 
- |<------------------------buffer----------------------------------->|
-                             |<------------compressed data---------->|
+ |<------------------------buffer--------------------------------->|
+                             |<-----------compressed data--------->|
  |<-----------decompressed size------------------>|
-                                                  |<-----margin----->|
+                                                  |<----margin---->|
 
  This technique is more useful for decompression,
  since decompressed size is typically larger,
- and margin is mostly required to avoid stripe overflow, so it's short.
+ and margin is short.
 
  In-place decompression will work inside any buffer
  which size is >= LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize).
  This presumes that decompressedSize > compressedSize.
  Otherwise, it means compression actually expanded data,
- which can happen when data is not compressible (already compressed, or encrypted),
  and it would be more efficient to store such data with a flag indicating it's not compressed.
+ This can happen when data is not compressible (already compressed, or encrypted).
 
- For compression, margin is larger, as it must be able to cope with both
+ For in-place compression, margin is larger, as it must be able to cope with both
  history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX,
  and data expansion, which can happen when input is not compressible.
  As a consequence, buffer size requirements are much higher,
@@ -388,10 +388,11 @@ int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
 
  There are ways to limit this cost for compression :
  - Reduce history size, by modifying LZ4_DISTANCE_MAX.
-   Note that it is a compile-time constant, so all future compression will apply this parameter.
+   Note that it is a compile-time constant, so all compressions will apply this limit.
    Lower values will reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX,
    so it's a reasonable trick when inputs are known to be small.
  - Require the compressor to deliver a "maximum compressed size".
+   This is the `dstCapacity` parameter in `LZ4_compress*()`.
    When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail,
    in which case, the return code will be 0 (zero).
    The caller must be ready for these cases to happen,
@@ -400,16 +401,16 @@ int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
  the amount of margin required for in-place compression.
 
  In-place compression can work in any buffer
- which size is >= LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize)
+ which size is >= (maxCompressedSize)
  with maxCompressedSize == LZ4_COMPRESSBOUND(srcSize) for guaranteed compression success.
- This macro depends on both maxCompressedSize and LZ4_DISTANCE_MAX,
- so there are ways to reduce memory requirements by playing with them.
+ LZ4_COMPRESS_INPLACE_BUFFER_SIZE() depends on both maxCompressedSize and LZ4_DISTANCE_MAX,
+ so it's possible to reduce memory requirements by playing with them.
  
 


 
-
#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize)   ( (decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize))  /**< note: presumes that compressedSize < decompressedSize */
+
#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize)   ((decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize))  /**< note: presumes that compressedSize < decompressedSize. note2: margin is overestimated a bit, since it could use compressedSize instead */
 


-
#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize)   ( (maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN)  /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */
+
#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize)   ((maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN)  /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */
 


 
PRIVATE DEFINITIONS
  Do not use these definitions directly.
diff --git a/lib/lz4.h b/lib/lz4.h
index 383c2db..0dfa637 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -476,23 +476,23 @@ LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const
  * and decompression to start at beginning of the buffer.
  * Buffer size must feature some margin, hence be larger than final size.
  *
- * |<------------------------buffer----------------------------------->|
- *                             |<------------compressed data---------->|
+ * |<------------------------buffer--------------------------------->|
+ *                             |<-----------compressed data--------->|
  * |<-----------decompressed size------------------>|
- *                                                  |<-----margin----->|
+ *                                                  |<----margin---->|
  *
  * This technique is more useful for decompression,
  * since decompressed size is typically larger,
- * and margin is mostly required to avoid stripe overflow, so it's short.
+ * and margin is short.
  *
  * In-place decompression will work inside any buffer
  * which size is >= LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize).
  * This presumes that decompressedSize > compressedSize.
  * Otherwise, it means compression actually expanded data,
- * which can happen when data is not compressible (already compressed, or encrypted),
  * and it would be more efficient to store such data with a flag indicating it's not compressed.
+ * This can happen when data is not compressible (already compressed, or encrypted).
  *
- * For compression, margin is larger, as it must be able to cope with both
+ * For in-place compression, margin is larger, as it must be able to cope with both
  * history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX,
  * and data expansion, which can happen when input is not compressible.
  * As a consequence, buffer size requirements are much higher,
@@ -500,10 +500,11 @@ LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const
  *
  * There are ways to limit this cost for compression :
  * - Reduce history size, by modifying LZ4_DISTANCE_MAX.
- *   Note that it is a compile-time constant, so all future compression will apply this parameter.
+ *   Note that it is a compile-time constant, so all compressions will apply this limit.
  *   Lower values will reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX,
  *   so it's a reasonable trick when inputs are known to be small.
  * - Require the compressor to deliver a "maximum compressed size".
+ *   This is the `dstCapacity` parameter in `LZ4_compress*()`.
  *   When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail,
  *   in which case, the return code will be 0 (zero).
  *   The caller must be ready for these cases to happen,
@@ -512,21 +513,21 @@ LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const
  * the amount of margin required for in-place compression.
  *
  * In-place compression can work in any buffer
- * which size is >= LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize)
+ * which size is >= (maxCompressedSize)
  * with maxCompressedSize == LZ4_COMPRESSBOUND(srcSize) for guaranteed compression success.
- * This macro depends on both maxCompressedSize and LZ4_DISTANCE_MAX,
- * so there are ways to reduce memory requirements by playing with them.
+ * LZ4_COMPRESS_INPLACE_BUFFER_SIZE() depends on both maxCompressedSize and LZ4_DISTANCE_MAX,
+ * so it's possible to reduce memory requirements by playing with them.
  */
 
-#define LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize)  (((decompressedSize) >> 8) + 32)
-#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize)   ( (decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize))  /**< note: presumes that compressedSize < decompressedSize */
+#define LZ4_DECOMPRESS_INPLACE_MARGIN(compressedSize)          (((compressedSize) >> 8) + 32)
+#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize)   ((decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize))  /**< note: presumes that compressedSize < decompressedSize. note2: margin is overestimated a bit, since it could use compressedSize instead */
 
 #ifndef LZ4_DISTANCE_MAX   /* history window size; can be user-defined at compile time */
 #  define LZ4_DISTANCE_MAX 65535   /* set to maximum value by default */
 #endif
 
-#define LZ4_COMPRESS_INPLACE_MARGIN (LZ4_DISTANCE_MAX + 32)
-#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize)   ( (maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN)  /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */
+#define LZ4_COMPRESS_INPLACE_MARGIN                           (LZ4_DISTANCE_MAX + 32)   /* LZ4_DISTANCE_MAX can be safely replaced by srcSize when it's smaller */
+#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize)   ((maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN)  /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */
 
 #endif   /* LZ4_STATIC_3504398509 */
 #endif   /* LZ4_STATIC_LINKING_ONLY */
-- 
cgit v0.12


From 99f1721ff5e3ab74fbe1fc8c68337ba87c1a8be9 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 31 May 2019 12:01:33 -0700
Subject: replaced while(1)

by for (;;)
just to please Visual Studio C4127 .
---
 tests/fullbench.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tests/fullbench.c b/tests/fullbench.c
index 2488a54..dbdb02c 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -358,7 +358,7 @@ static int local_LZ4F_decompress_followHint(const char* src, char* dst, int srcS
     size_t outPos = 0;
     size_t outRemaining = maxOutSize - outPos;
 
-    do {
+    for (;;) {
         size_t const sizeHint = LZ4F_decompress(g_dCtx, dst+outPos, &outRemaining, src+inPos, &inSize, NULL);
         assert(!LZ4F_isError(sizeHint));
 
@@ -369,8 +369,7 @@ static int local_LZ4F_decompress_followHint(const char* src, char* dst, int srcS
         outRemaining = maxOutSize - outPos;
 
         if (!sizeHint) break;
-
-    } while(1);
+    }
 
     /* frame completed */
     if (inPos != totalInSize) {
-- 
cgit v0.12


From 33a04fb8bd19fb4450ea37dad6aa0fd0d7f4007c Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 31 May 2019 13:25:12 -0700
Subject: fullbench: ensure decompressionFunction and dName are initialized

Visual Studio seems to miss that they are necessarily initialized in the switch() { case: }
---
 tests/fullbench.c | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/tests/fullbench.c b/tests/fullbench.c
index dbdb02c..4609f13 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -571,9 +571,15 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
 
           nbChunks = (int) (((int)benchedSize + (g_chunkSize-1))/ g_chunkSize);
           for (i=0; i g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }
+              if ((int)remaining > g_chunkSize) {
+                  chunkP[i].origSize = g_chunkSize;
+                  remaining -= (size_t)g_chunkSize;
+              } else {
+                  chunkP[i].origSize = (int)remaining;
+                  remaining = 0;
+              }
               chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;
               chunkP[i].compressedSize = 0;
           }
@@ -586,8 +592,8 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
 
         /* Decompression Algorithms */
         for (dAlgNb=0; (dAlgNb <= NB_DECOMPRESSION_ALGORITHMS) && g_decompressionTest; dAlgNb++) {
-            const char* dName;
-            int (*decompressionFunction)(const char*, char*, int, int);
+            const char* dName = NULL;
+            int (*decompressionFunction)(const char*, char*, int, int) = NULL;
             double bestTime = 100000000.;
             int checkResult = 1;
 
@@ -609,6 +615,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
             case 11:
                 if (dAlgNb == 10) { decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress"; }  /* can be skipped */
                 if (dAlgNb == 11) { decompressionFunction = local_LZ4F_decompress_followHint; dName = "LZ4F_decompress_followHint"; }  /* can be skipped */
+                /* prepare compressed data using frame format */
                 {   size_t const fcsize = LZ4F_compressFrame(compressed_buff, (size_t)compressedBuffSize, orig_buff, benchedSize, NULL);
                     assert(!LZ4F_isError(fcsize));
                     chunkP[0].origSize = (int)benchedSize;
@@ -620,6 +627,9 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
                 continue;   /* skip if unknown ID */
             }
 
+            assert(decompressionFunction != NULL);
+            assert(dName != NULL);
+
             { size_t i; for (i=0; i
Date: Tue, 4 Jun 2019 14:04:49 -0700
Subject: restored FORCE_INLINE

---
 lib/lz4.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 076565a..cac3240 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -120,7 +120,6 @@
 #  pragma warning(disable : 4293)        /* disable: C4293: too large shift (32-bits) */
 #endif  /* _MSC_VER */
 
-#define LZ4_FORCE_INLINE static
 #ifndef LZ4_FORCE_INLINE
 #  ifdef _MSC_VER    /* Visual Studio */
 #    define LZ4_FORCE_INLINE static __forceinline
-- 
cgit v0.12


From f7b52ecbbad2d2632e0ae08f105d38e3f49a3673 Mon Sep 17 00:00:00 2001
From: Ephraim Park 
Date: Tue, 4 Jun 2019 12:45:31 -0700
Subject: circleci : use custom docker image with pre-installed dependencies

---
 .circleci/config.yml                | 33 +--------------------------------
 .circleci/images/primary/Dockerfile | 12 ++++++++++++
 2 files changed, 13 insertions(+), 32 deletions(-)
 create mode 100644 .circleci/images/primary/Dockerfile

diff --git a/.circleci/config.yml b/.circleci/config.yml
index a620cd1..3abcbc1 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -27,8 +27,7 @@ jobs:
     # To see the list of pre-built images that CircleCI provides for most common languages see
     # https://circleci.com/docs/2.0/circleci-images/
     docker:
-    - image: circleci/build-image:ubuntu-14.04-XXL-upstart-1189-5614f37
-      command: /sbin/init
+    - image: fbopensource/lz4-circleci-primary:0.0.4
     steps:
     # Machine Setup
     #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
@@ -38,36 +37,6 @@ jobs:
     # In many cases you can simplify this from what is generated here.
     # 'See docs on artifact collection here https://circleci.com/docs/2.0/artifacts/'
     - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS
-    # Dependencies
-    #   This would typically go in either a build or a build-and-test job when using workflows
-    # Restore the dependency cache
-    - restore_cache:
-        keys:
-        # This branch if available
-        - v1-dep-{{ .Branch }}-
-        # Default branch if not
-        - v1-dep-dev-
-        # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly
-        - v1-dep-
-    # This is based on your 1.0 configuration file or project settings
-    - run: sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update
-    - run: sudo apt-get -y install qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu
-    - run: sudo apt-get -y install qemu-system-arm gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
-    - run: sudo apt-get -y install libc6-dev-i386 clang gcc-5 gcc-5-multilib gcc-6 valgrind
-    # Save dependency cache
-    - save_cache:
-        key: v1-dep-{{ .Branch }}-{{ epoch }}
-        paths:
-        # This is a broad list of cache paths to include many possible development environments
-        # You can probably delete some of these entries
-        - vendor/bundle
-        - ~/virtualenvs
-        - ~/.m2
-        - ~/.ivy2
-        - ~/.bundle
-        - ~/.go_workspace
-        - ~/.gradle
-        - ~/.cache/bower
     # Test
     #   This would typically be a build job when using workflows, possibly combined with build
     # This is based on your 1.0 configuration file or project settings
diff --git a/.circleci/images/primary/Dockerfile b/.circleci/images/primary/Dockerfile
new file mode 100644
index 0000000..7767014
--- /dev/null
+++ b/.circleci/images/primary/Dockerfile
@@ -0,0 +1,12 @@
+FROM circleci/buildpack-deps:bionic
+
+RUN sudo apt-get -y -qq update
+RUN sudo apt-get -y install software-properties-common
+RUN sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
+RUN sudo apt-get -y install cmake
+RUN sudo apt-get -y install qemu-system-ppc qemu-user-static qemu-system-arm
+RUN sudo apt-get -y install libc6-dev-armel-cross  libc6-dev-arm64-cross libc6-dev-i386
+RUN sudo apt-get -y install clang clang-tools
+RUN sudo apt-get -y install gcc-5 gcc-5-multilib gcc-6
+RUN sudo apt-get -y install valgrind
+RUN sudo apt-get -y install gcc-multilib-powerpc-linux-gnu gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi gcc-aarch64-linux-gnu
-- 
cgit v0.12


From 1d759576b92a54a29e549bb3a4d42f0660ab4b31 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 6 Jun 2019 13:20:30 -0700
Subject: precise again that LZ4 decoder needs metadata

and that such metadata must be provided / sent / saved by the application.
---
 doc/lz4_manual.html | 51 +++++++++++++++++++++++++++++++--------------------
 lib/lz4.h           | 53 +++++++++++++++++++++++++++++++----------------------
 2 files changed, 62 insertions(+), 42 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index 1480089..18daa05 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -21,7 +21,7 @@
 
 

 
Introduction
-  LZ4 is lossless compression algorithm, providing compression speed at 500 MB/s per core,
+  LZ4 is lossless compression algorithm, providing compression speed >500 MB/s per core,
   scalable with multi-cores CPU. It features an extremely fast decoder, with speed in
   multiple GB/s per core, typically reaching RAM speed limits on multi-core systems.
 
@@ -33,7 +33,10 @@
     - unbounded multiple steps (described as Streaming compression)
 
   lz4.h generates and decodes LZ4-compressed blocks (doc/lz4_Block_format.md).
-  Decompressing a block requires additional metadata, such as its compressed size.
+  Decompressing such a compressed block requires additional metadata.
+  Exact metadata depends on exact decompression function.
+  For the typical case of LZ4_decompress_safe(),
+  metadata includes block's compressed size, and maximum bound of decompressed size.
   Each application is free to encode and pass such metadata in whichever way it wants.
 
   lz4.h only handle blocks, it can not generate Frames.
@@ -66,27 +69,35 @@
 
Simple Functions

 
 
int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity);
-
    Compresses 'srcSize' bytes from buffer 'src'
-    into already allocated 'dst' buffer of size 'dstCapacity'.
-    Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize).
-    It also runs faster, so it's a recommended setting.
-    If the function cannot compress 'src' into a more limited 'dst' budget,
-    compression stops *immediately*, and the function result is zero.
-    In which case, 'dst' content is undefined (invalid).
-        srcSize : max supported value is LZ4_MAX_INPUT_SIZE.
-        dstCapacity : size of buffer 'dst' (which must be already allocated)
-       @return  : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity)
-                  or 0 if compression fails
-    Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer).
+
  Compresses 'srcSize' bytes from buffer 'src'
+  into already allocated 'dst' buffer of size 'dstCapacity'.
+  Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize).
+  It also runs faster, so it's a recommended setting.
+  If the function cannot compress 'src' into a more limited 'dst' budget,
+  compression stops *immediately*, and the function result is zero.
+  In which case, 'dst' content is undefined (invalid).
+      srcSize : max supported value is LZ4_MAX_INPUT_SIZE.
+      dstCapacity : size of buffer 'dst' (which must be already allocated)
+     @return  : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity)
+                or 0 if compression fails
+ Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer).
+ 
 


 
 
int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity);
-
    compressedSize : is the exact complete size of the compressed block.
-    dstCapacity : is the size of destination buffer, which must be already allocated.
-   @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity)
-             If destination buffer is not large enough, decoding will stop and output an error code (negative value).
-             If the source stream is detected malformed, the function will stop decoding and return a negative result.
-    Note : This function is protected against malicious data packets (never writes outside 'dst' buffer, nor read outside 'source' buffer).
+
  compressedSize : is the exact complete size of the compressed block.
+  dstCapacity : is the size of destination buffer (which must be already allocated), presumed an upper bound of decompressed size.
+ @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity)
+           If destination buffer is not large enough, decoding will stop and output an error code (negative value).
+           If the source stream is detected malformed, the function will stop decoding and return a negative result.
+ Note 1 : This function is protected against malicious data packets :
+          it will never writes outside 'dst' buffer, nor read outside 'source' buffer,
+          even if the compressed block is maliciously modified to order the decoder to do these actions.
+          In such case, the decoder stops immediately, and considers the compressed block malformed.
+ Note 2 : compressedSize and dstCapacity must be provided to the function, the compressed block does not contain them.
+          The implementation is free to send / store / derive this information in whichever way is most beneficial.
+          If there is a need for a different format which bundles together both compressed data and its metadata, consider looking at lz4frame.h instead.
+ 
 


 
 
Advanced Functions

diff --git a/lib/lz4.h b/lib/lz4.h
index 0dfa637..ad93d00 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -46,7 +46,7 @@ extern "C" {
 /**
   Introduction
 
-  LZ4 is lossless compression algorithm, providing compression speed at 500 MB/s per core,
+  LZ4 is lossless compression algorithm, providing compression speed >500 MB/s per core,
   scalable with multi-cores CPU. It features an extremely fast decoder, with speed in
   multiple GB/s per core, typically reaching RAM speed limits on multi-core systems.
 
@@ -58,7 +58,10 @@ extern "C" {
     - unbounded multiple steps (described as Streaming compression)
 
   lz4.h generates and decodes LZ4-compressed blocks (doc/lz4_Block_format.md).
-  Decompressing a block requires additional metadata, such as its compressed size.
+  Decompressing such a compressed block requires additional metadata.
+  Exact metadata depends on exact decompression function.
+  For the typical case of LZ4_decompress_safe(),
+  metadata includes block's compressed size, and maximum bound of decompressed size.
   Each application is free to encode and pass such metadata in whichever way it wants.
 
   lz4.h only handle blocks, it can not generate Frames.
@@ -129,29 +132,35 @@ LZ4LIB_API const char* LZ4_versionString (void);   /**< library version string;
 *  Simple Functions
 **************************************/
 /*! LZ4_compress_default() :
-    Compresses 'srcSize' bytes from buffer 'src'
-    into already allocated 'dst' buffer of size 'dstCapacity'.
-    Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize).
-    It also runs faster, so it's a recommended setting.
-    If the function cannot compress 'src' into a more limited 'dst' budget,
-    compression stops *immediately*, and the function result is zero.
-    In which case, 'dst' content is undefined (invalid).
-        srcSize : max supported value is LZ4_MAX_INPUT_SIZE.
-        dstCapacity : size of buffer 'dst' (which must be already allocated)
-       @return  : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity)
-                  or 0 if compression fails
-    Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer).
-*/
+ *  Compresses 'srcSize' bytes from buffer 'src'
+ *  into already allocated 'dst' buffer of size 'dstCapacity'.
+ *  Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize).
+ *  It also runs faster, so it's a recommended setting.
+ *  If the function cannot compress 'src' into a more limited 'dst' budget,
+ *  compression stops *immediately*, and the function result is zero.
+ *  In which case, 'dst' content is undefined (invalid).
+ *      srcSize : max supported value is LZ4_MAX_INPUT_SIZE.
+ *      dstCapacity : size of buffer 'dst' (which must be already allocated)
+ *     @return  : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity)
+ *                or 0 if compression fails
+ * Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer).
+ */
 LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity);
 
 /*! LZ4_decompress_safe() :
-    compressedSize : is the exact complete size of the compressed block.
-    dstCapacity : is the size of destination buffer, which must be already allocated.
-   @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity)
-             If destination buffer is not large enough, decoding will stop and output an error code (negative value).
-             If the source stream is detected malformed, the function will stop decoding and return a negative result.
-    Note : This function is protected against malicious data packets (never writes outside 'dst' buffer, nor read outside 'source' buffer).
-*/
+ *  compressedSize : is the exact complete size of the compressed block.
+ *  dstCapacity : is the size of destination buffer (which must be already allocated), presumed an upper bound of decompressed size.
+ * @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity)
+ *           If destination buffer is not large enough, decoding will stop and output an error code (negative value).
+ *           If the source stream is detected malformed, the function will stop decoding and return a negative result.
+ * Note 1 : This function is protected against malicious data packets :
+ *          it will never writes outside 'dst' buffer, nor read outside 'source' buffer,
+ *          even if the compressed block is maliciously modified to order the decoder to do these actions.
+ *          In such case, the decoder stops immediately, and considers the compressed block malformed.
+ * Note 2 : compressedSize and dstCapacity must be provided to the function, the compressed block does not contain them.
+ *          The implementation is free to send / store / derive this information in whichever way is most beneficial.
+ *          If there is a need for a different format which bundles together both compressed data and its metadata, consider looking at lz4frame.h instead.
+ */
 LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity);
 
 
-- 
cgit v0.12


From 798301b4e144fab5d25fc34566c1419685f5f1eb Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 6 Jun 2019 14:17:44 -0700
Subject: update simple_buffer example

there were a few tiny inaccuracies, especially in error conditions.
---
 examples/simple_buffer.c | 45 +++++++++++++++++++++++++--------------------
 1 file changed, 25 insertions(+), 20 deletions(-)

diff --git a/examples/simple_buffer.c b/examples/simple_buffer.c
index 54e542a..027fc85 100644
--- a/examples/simple_buffer.c
+++ b/examples/simple_buffer.c
@@ -3,18 +3,18 @@
  * Copyright  : Kyle Harper
  * License    : Follows same licensing as the lz4.c/lz4.h program at any given time.  Currently, BSD 2.
  * Description: Example program to demonstrate the basic usage of the compress/decompress functions within lz4.c/lz4.h.
- *              The functions you'll likely want are LZ4_compress_default and LZ4_decompress_safe.  Both of these are documented in
- *              the lz4.h header file; I recommend reading them.
+ *              The functions you'll likely want are LZ4_compress_default and LZ4_decompress_safe.
+ *              Both of these are documented in the lz4.h header file; I recommend reading them.
  */
 
-/* Includes, for Power! */
-#include "lz4.h"     // This is all that is required to expose the prototypes for basic compression and decompression.
+/* Dependencies */
 #include    // For printf()
 #include   // For memcmp()
 #include   // For exit()
+#include "lz4.h"     // This is all that is required to expose the prototypes for basic compression and decompression.
 
 /*
- * Easy show-error-and-bail function.
+ * Simple show-error-and-bail function.
  */
 void run_screaming(const char* message, const int code) {
   printf("%s \n", message);
@@ -32,9 +32,9 @@ int main(void) {
   //   1) The return codes of LZ4_ functions are important.
   //      Read lz4.h if you're unsure what a given code means.
   //   2) LZ4 uses char* pointers in all LZ4_ functions.
-  //      This is baked into the API and probably not going to change.
-  //      If your program uses pointers that are unsigned char*, void*, or otherwise different,
-  //      you may need to do some casting or set the right -W compiler flags to ignore those warnings (e.g.: -Wno-pointer-sign).
+  //      This is baked into the API and not going to change, for consistency.
+  //      If your program uses different pointer types,
+  //      you may need to do some casting or set the right -Wno compiler flags to ignore those warnings (e.g.: -Wno-pointer-sign).
 
   /* Compression */
   // We'll store some text into a variable pointed to by *src to be compressed later.
@@ -44,7 +44,7 @@ int main(void) {
   // LZ4 provides a function that will tell you the maximum size of compressed output based on input data via LZ4_compressBound().
   const int max_dst_size = LZ4_compressBound(src_size);
   // We will use that size for our destination boundary when allocating space.
-  char* compressed_data = malloc(max_dst_size);
+  char* compressed_data = malloc((size_t)max_dst_size);
   if (compressed_data == NULL)
     run_screaming("Failed to allocate memory for *compressed_data.", 1);
   // That's all the information and preparation LZ4 needs to compress *src into *compressed_data.
@@ -52,21 +52,26 @@ int main(void) {
   // Save the return value for error checking.
   const int compressed_data_size = LZ4_compress_default(src, compressed_data, src_size, max_dst_size);
   // Check return_value to determine what happened.
-  if (compressed_data_size < 0)
-    run_screaming("A negative result from LZ4_compress_default indicates a failure trying to compress the data.  See exit code (echo $?) for value returned.", compressed_data_size);
-  if (compressed_data_size == 0)
-    run_screaming("A result of 0 means compression worked, but was stopped because the destination buffer couldn't hold all the information.", 1);
+  if (compressed_data_size <= 0)
+    run_screaming("A 0 or negative result from LZ4_compress_default() indicates a failure trying to compress the data. ", 1);
   if (compressed_data_size > 0)
     printf("We successfully compressed some data!\n");
   // Not only does a positive return_value mean success, the value returned == the number of bytes required.
   // You can use this to realloc() *compress_data to free up memory, if desired.  We'll do so just to demonstrate the concept.
-  compressed_data = (char *)realloc(compressed_data, compressed_data_size);
+  compressed_data = (char *)realloc(compressed_data, (size_t)compressed_data_size);
   if (compressed_data == NULL)
     run_screaming("Failed to re-alloc memory for compressed_data.  Sad :(", 1);
 
+
   /* Decompression */
-  // Now that we've successfully compressed the information from *src to *compressed_data, let's do the opposite!  We'll create a
-  // *new_src location of size src_size since we know that value.
+  // Now that we've successfully compressed the information from *src to *compressed_data, let's do the opposite!
+  // The decompression will need to know the compressed size, and an upper bound of the decompressed size.
+  // In this example, we just re-use this information from previous section,
+  // but in a real-world scenario, metadata must be transmitted to the decompression side.
+  // Each implementation is in charge of this part. Oftentimes, it adds some header of its own.
+  // Sometimes, the metadata can be extracted from the local context.
+
+  // First, let's create a *new_src location of size src_size since we know that value.
   char* const regen_buffer = malloc(src_size);
   if (regen_buffer == NULL)
     run_screaming("Failed to allocate memory for *regen_buffer.", 1);
@@ -77,17 +82,17 @@ int main(void) {
   free(compressed_data);   /* no longer useful */
   if (decompressed_size < 0)
     run_screaming("A negative result from LZ4_decompress_safe indicates a failure trying to decompress the data.  See exit code (echo $?) for value returned.", decompressed_size);
-  if (decompressed_size == 0)
-    run_screaming("I'm not sure this function can ever return 0.  Documentation in lz4.h doesn't indicate so.", 1);
-  if (decompressed_size > 0)
+  if (decompressed_size >= 0)
     printf("We successfully decompressed some data!\n");
   // Not only does a positive return value mean success,
   // value returned == number of bytes regenerated from compressed_data stream.
+  if (decompressed_size != src_size)
+    run_screaming("Decompressed data is different from original! \n", 1);
 
   /* Validation */
   // We should be able to compare our original *src with our *new_src and be byte-for-byte identical.
   if (memcmp(src, regen_buffer, src_size) != 0)
     run_screaming("Validation failed.  *src and *new_src are not identical.", 1);
-  printf("Validation done.  The string we ended up with is:\n%s\n", regen_buffer);
+  printf("Validation done. The string we ended up with is:\n%s\n", regen_buffer);
   return 0;
 }
-- 
cgit v0.12


From baf9b0e0434e1e516f468c4b96bde5bb38d4c737 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 24 Jun 2019 16:08:30 -0700
Subject: fix #734 : --version should output to stdout

instead of stderr
---
 programs/lz4cli.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index 3619cd5..3b5d61e 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -67,6 +67,7 @@ static int g_lz4c_legacy_commands = 0;
 /*-************************************
 *  Macros
 ***************************************/
+#define DISPLAYOUT(...)        fprintf(stdout, __VA_ARGS__)
 #define DISPLAY(...)           fprintf(stderr, __VA_ARGS__)
 #define DISPLAYLEVEL(l, ...)   if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
 static unsigned displayLevel = 2;   /* 0 : no display ; 1: errors only ; 2 : downgradable normal ; 3 : non-downgradable normal; 4 : + information */
@@ -390,7 +391,7 @@ int main(int argc, const char** argv)
                 if (!strcmp(argument,  "--favor-decSpeed")) { LZ4IO_favorDecSpeed(prefs, 1); continue; }
                 if (!strcmp(argument,  "--verbose")) { displayLevel++; continue; }
                 if (!strcmp(argument,  "--quiet")) { if (displayLevel) displayLevel--; continue; }
-                if (!strcmp(argument,  "--version")) { DISPLAY(WELCOME_MESSAGE); return 0; }
+                if (!strcmp(argument,  "--version")) { DISPLAYOUT(WELCOME_MESSAGE); return 0; }
                 if (!strcmp(argument,  "--help")) { usage_advanced(exeName); goto _cleanup; }
                 if (!strcmp(argument,  "--keep")) { LZ4IO_setRemoveSrcFile(prefs, 0); continue; }   /* keep source file (default) */
                 if (!strcmp(argument,  "--rm")) { LZ4IO_setRemoveSrcFile(prefs, 1); continue; }
@@ -437,7 +438,7 @@ int main(int argc, const char** argv)
                 switch(argument[0])
                 {
                     /* Display help */
-                case 'V': DISPLAY(WELCOME_MESSAGE); goto _cleanup;   /* Version */
+                case 'V': DISPLAYOUT(WELCOME_MESSAGE); goto _cleanup;   /* Version */
                 case 'h': usage_advanced(exeName); goto _cleanup;
                 case 'H': usage_longhelp(exeName); goto _cleanup;
 
-- 
cgit v0.12


From a5cf079d4dc9097c4e58f0eb7b0996b0a6d91696 Mon Sep 17 00:00:00 2001
From: Max Dymond 
Date: Tue, 25 Jun 2019 17:22:02 +0100
Subject: Add a fuzzing target that compiles in the oss-fuzz environment

---
 .travis.yml                 |  6 ++++
 Makefile                    |  1 +
 ossfuzz/Makefile            | 54 +++++++++++++++++++++++++++++++++
 ossfuzz/compress_fuzzer.cc  | 22 ++++++++++++++
 ossfuzz/ossfuzz.sh          | 26 ++++++++++++++++
 ossfuzz/standaloneengine.cc | 74 +++++++++++++++++++++++++++++++++++++++++++++
 ossfuzz/testinput.h         |  3 ++
 ossfuzz/travisoss.sh        | 24 +++++++++++++++
 8 files changed, 210 insertions(+)
 create mode 100644 ossfuzz/Makefile
 create mode 100644 ossfuzz/compress_fuzzer.cc
 create mode 100755 ossfuzz/ossfuzz.sh
 create mode 100644 ossfuzz/standaloneengine.cc
 create mode 100644 ossfuzz/testinput.h
 create mode 100755 ossfuzz/travisoss.sh

diff --git a/.travis.yml b/.travis.yml
index ee643e5..4d45e89 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -194,5 +194,11 @@ matrix:
         - pushd build
         - DESTDIR=./staging ninja install
         - tree ./staging
+
+    # oss-fuzz compilation test
+    - name: Compile OSS-Fuzz targets
+      script:
+        - ./ossfuzz/travisoss.sh
+
   allow_failures:
     - env: ALLOW_FAILURES=true
diff --git a/Makefile b/Makefile
index e24cec5..34835fd 100644
--- a/Makefile
+++ b/Makefile
@@ -34,6 +34,7 @@ LZ4DIR  = lib
 PRGDIR  = programs
 TESTDIR = tests
 EXDIR   = examples
+FUZZDIR = ossfuzz
 
 include Makefile.inc
 
diff --git a/ossfuzz/Makefile b/ossfuzz/Makefile
new file mode 100644
index 0000000..94829b2
--- /dev/null
+++ b/ossfuzz/Makefile
@@ -0,0 +1,54 @@
+# ##########################################################################
+# LZ4 oss fuzzer - Makefile
+#
+# GPL v2 License
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# You can contact the author at :
+#  - LZ4 homepage : http://www.lz4.org
+#  - LZ4 source repository : https://github.com/lz4/lz4
+# ##########################################################################
+# lz4_fuzzer : OSS Fuzz test tool
+# ##########################################################################
+
+LZ4DIR  := ../lib
+LIB_FUZZING_ENGINE ?= standaloneengine.o
+
+DEBUGLEVEL?= 1
+DEBUGFLAGS = -g -DLZ4_DEBUG=$(DEBUGLEVEL)
+
+CFLAGS  += -I$(LZ4DIR) $(DEBUGFLAGS) $(MOREFLAGS)
+CPPFLAGS+= -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_
+FLAGS    = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
+
+include ../Makefile.inc
+
+# Include a rule to build the static library if calling this target
+# directly.
+$(LZ4DIR)/liblz4.a:
+	$(MAKE) -C $(LZ4DIR) CFLAGS="$(CFLAGS)" liblz4.a
+
+%.o: %.cc
+	$(CXX) -c $(CFLAGS) $(CPPFLAGS) $< -o $@
+
+.PHONY: compress_fuzzer
+compress_fuzzer: compress_fuzzer.o $(LZ4DIR)/liblz4.a
+	# Compile the standalone code just in case. The OSS-Fuzz code might
+	# override the LIB_FUZZING_ENGINE value to "-fsanitize=fuzzer"
+	$(CXX) -c $(CFLAGS) $(CPPFLAGS) standaloneengine.cc -o standaloneengine.o
+
+	# Now compile the actual fuzzer.
+	$(CXX) $(FLAGS) $(LIB_FUZZING_ENGINE) $^ -o $@$(EXT)
diff --git a/ossfuzz/compress_fuzzer.cc b/ossfuzz/compress_fuzzer.cc
new file mode 100644
index 0000000..006a0ab
--- /dev/null
+++ b/ossfuzz/compress_fuzzer.cc
@@ -0,0 +1,22 @@
+#include 
+#include 
+#include 
+#include "lz4.h"
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
+{
+  size_t const compressed_dest_size = LZ4_compressBound(size);
+  char *const dest_buffer = (char *)malloc(compressed_dest_size);
+
+  int result = LZ4_compress_default((const char*)data, dest_buffer,
+                                    size, compressed_dest_size);
+
+  if (result == 0)
+  {
+    abort();
+  }
+
+  free(dest_buffer);
+
+  return 0;
+}
diff --git a/ossfuzz/ossfuzz.sh b/ossfuzz/ossfuzz.sh
new file mode 100755
index 0000000..e0cb63c
--- /dev/null
+++ b/ossfuzz/ossfuzz.sh
@@ -0,0 +1,26 @@
+#!/bin/bash -eu
+
+# This script is called by the oss-fuzz main project when compiling the fuzz
+# targets. This script is regression tested by travisoss.sh.
+
+# Save off the current folder as the build root.
+export BUILD_ROOT=$PWD
+
+# lz4 uses CPPFLAGS rather than CXX flags.
+export CPPFLAGS="${CXXFLAGS}"
+
+echo "CC: $CC"
+echo "CXX: $CXX"
+echo "LIB_FUZZING_ENGINE: $LIB_FUZZING_ENGINE"
+echo "CFLAGS: $CFLAGS"
+echo "CPPFLAGS: $CPPFLAGS"
+echo "OUT: $OUT"
+
+export MAKEFLAGS+="-j$(nproc)"
+
+pushd ossfuzz
+make V=1 compress_fuzzer
+popd
+
+# Copy the fuzzers to the target directory.
+cp -v ossfuzz/compress_fuzzer $OUT/
diff --git a/ossfuzz/standaloneengine.cc b/ossfuzz/standaloneengine.cc
new file mode 100644
index 0000000..175360e
--- /dev/null
+++ b/ossfuzz/standaloneengine.cc
@@ -0,0 +1,74 @@
+#include 
+#include 
+#include 
+
+#include "testinput.h"
+
+/**
+ * Main procedure for standalone fuzzing engine.
+ *
+ * Reads filenames from the argument array. For each filename, read the file
+ * into memory and then call the fuzzing interface with the data.
+ */
+int main(int argc, char **argv)
+{
+  int ii;
+  for(ii = 1; ii < argc; ii++)
+  {
+    FILE *infile;
+    printf("[%s] ", argv[ii]);
+
+    /* Try and open the file. */
+    infile = fopen(argv[ii], "rb");
+    if(infile)
+    {
+      uint8_t *buffer = NULL;
+      size_t buffer_len;
+
+      printf("Opened.. ");
+
+      /* Get the length of the file. */
+      fseek(infile, 0L, SEEK_END);
+      buffer_len = ftell(infile);
+
+      /* Reset the file indicator to the beginning of the file. */
+      fseek(infile, 0L, SEEK_SET);
+
+      /* Allocate a buffer for the file contents. */
+      buffer = (uint8_t *)calloc(buffer_len, sizeof(uint8_t));
+      if(buffer)
+      {
+        /* Read all the text from the file into the buffer. */
+        fread(buffer, sizeof(uint8_t), buffer_len, infile);
+        printf("Read %zu bytes, fuzzing.. ", buffer_len);
+
+        /* Call the fuzzer with the data. */
+        LLVMFuzzerTestOneInput(buffer, buffer_len);
+
+        printf("complete !!");
+
+        /* Free the buffer as it's no longer needed. */
+        free(buffer);
+        buffer = NULL;
+      }
+      else
+      {
+        fprintf(stderr,
+                "[%s] Failed to allocate %zu bytes \n",
+                argv[ii],
+                buffer_len);
+      }
+
+      /* Close the file as it's no longer needed. */
+      fclose(infile);
+      infile = NULL;
+    }
+    else
+    {
+      /* Failed to open the file. Maybe wrong name or wrong permissions? */
+      fprintf(stderr, "[%s] Open failed. \n", argv[ii]);
+    }
+
+    printf("\n");
+  }
+}
diff --git a/ossfuzz/testinput.h b/ossfuzz/testinput.h
new file mode 100644
index 0000000..6ab9b51
--- /dev/null
+++ b/ossfuzz/testinput.h
@@ -0,0 +1,3 @@
+#include 
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size);
diff --git a/ossfuzz/travisoss.sh b/ossfuzz/travisoss.sh
new file mode 100755
index 0000000..3b2f26f
--- /dev/null
+++ b/ossfuzz/travisoss.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -ex
+
+# Clone the oss-fuzz repository
+git clone https://github.com/google/oss-fuzz.git /tmp/ossfuzz
+
+if [[ ! -d /tmp/ossfuzz/projects/lz4 ]]
+then
+    echo "Could not find the lz4 project in ossfuzz"
+
+    # Exit with a success code while the lz4 project is not expected to exist
+    # on oss-fuzz.
+    exit 0
+fi
+
+# Modify the oss-fuzz Dockerfile so that we're checking out the current branch on travis.
+sed -i "s@https://github.com/lz4/lz4.git@-b $TRAVIS_BRANCH https://github.com/lz4/lz4.git@" /tmp/ossfuzz/projects/lz4/Dockerfile
+
+# Try and build the fuzzers
+pushd /tmp/ossfuzz
+python infra/helper.py build_image --pull lz4
+python infra/helper.py build_fuzzers lz4
+popd
-- 
cgit v0.12


From 88a7cfd7283ea9c51fd044b9f58aee47b9ed3d16 Mon Sep 17 00:00:00 2001
From: Max Dymond 
Date: Fri, 28 Jun 2019 20:54:46 +0100
Subject: Code review markups:

- Correct use of CPPFLAGS
- Detect allocation failure
- Add a CHECK macro for failure
---
 ossfuzz/Makefile           | 12 ++++++------
 ossfuzz/compress_fuzzer.cc | 18 +++++++++++-------
 ossfuzz/ossfuzz.sh         |  5 +----
 3 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/ossfuzz/Makefile b/ossfuzz/Makefile
index 94829b2..2a7e439 100644
--- a/ossfuzz/Makefile
+++ b/ossfuzz/Makefile
@@ -30,9 +30,9 @@ LIB_FUZZING_ENGINE ?= standaloneengine.o
 DEBUGLEVEL?= 1
 DEBUGFLAGS = -g -DLZ4_DEBUG=$(DEBUGLEVEL)
 
-CFLAGS  += -I$(LZ4DIR) $(DEBUGFLAGS) $(MOREFLAGS)
-CPPFLAGS+= -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_
-FLAGS    = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
+CFLAGS   += -I$(LZ4DIR) $(DEBUGFLAGS) $(MOREFLAGS)
+CXXFLAGS += -I$(LZ4DIR) $(DEBUGFLAGS) $(MOREFLAGS)
+CPPFLAGS += -DXXH_NAMESPACE=LZ4_
 
 include ../Makefile.inc
 
@@ -42,13 +42,13 @@ $(LZ4DIR)/liblz4.a:
 	$(MAKE) -C $(LZ4DIR) CFLAGS="$(CFLAGS)" liblz4.a
 
 %.o: %.cc
-	$(CXX) -c $(CFLAGS) $(CPPFLAGS) $< -o $@
+	$(CXX) -c $(CXXFLAGS) $(CPPFLAGS) $< -o $@
 
 .PHONY: compress_fuzzer
 compress_fuzzer: compress_fuzzer.o $(LZ4DIR)/liblz4.a
 	# Compile the standalone code just in case. The OSS-Fuzz code might
 	# override the LIB_FUZZING_ENGINE value to "-fsanitize=fuzzer"
-	$(CXX) -c $(CFLAGS) $(CPPFLAGS) standaloneengine.cc -o standaloneengine.o
+	$(CXX) -c $(CXXFLAGS) $(CPPFLAGS) standaloneengine.cc -o standaloneengine.o
 
 	# Now compile the actual fuzzer.
-	$(CXX) $(FLAGS) $(LIB_FUZZING_ENGINE) $^ -o $@$(EXT)
+	$(CXX) $(CXXFLAGS) $(CPPFLAGS) $(LDFLAGS) $(LIB_FUZZING_ENGINE) $^ -o $@$(EXT)
diff --git a/ossfuzz/compress_fuzzer.cc b/ossfuzz/compress_fuzzer.cc
index 006a0ab..4a720e2 100644
--- a/ossfuzz/compress_fuzzer.cc
+++ b/ossfuzz/compress_fuzzer.cc
@@ -3,20 +3,24 @@
 #include 
 #include "lz4.h"
 
+#define CHECK(COND)   if (!(COND)) { abort(); }
+
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 {
   size_t const compressed_dest_size = LZ4_compressBound(size);
   char *const dest_buffer = (char *)malloc(compressed_dest_size);
 
-  int result = LZ4_compress_default((const char*)data, dest_buffer,
-                                    size, compressed_dest_size);
-
-  if (result == 0)
+  if (dest_buffer != NULL)
   {
-    abort();
-  }
+    // Allocation succeeded, try compressing the incoming data.
+    int result = LZ4_compress_default((const char*)data,
+                                      dest_buffer,
+                                      size,
+                                      compressed_dest_size);
+    CHECK(result != 0);
 
-  free(dest_buffer);
+    free(dest_buffer);
+  }
 
   return 0;
 }
diff --git a/ossfuzz/ossfuzz.sh b/ossfuzz/ossfuzz.sh
index e0cb63c..87bc213 100755
--- a/ossfuzz/ossfuzz.sh
+++ b/ossfuzz/ossfuzz.sh
@@ -6,14 +6,11 @@
 # Save off the current folder as the build root.
 export BUILD_ROOT=$PWD
 
-# lz4 uses CPPFLAGS rather than CXX flags.
-export CPPFLAGS="${CXXFLAGS}"
-
 echo "CC: $CC"
 echo "CXX: $CXX"
 echo "LIB_FUZZING_ENGINE: $LIB_FUZZING_ENGINE"
 echo "CFLAGS: $CFLAGS"
-echo "CPPFLAGS: $CPPFLAGS"
+echo "CXXFLAGS: $CXXFLAGS"
 echo "OUT: $OUT"
 
 export MAKEFLAGS+="-j$(nproc)"
-- 
cgit v0.12


From 60d71dc20c5f9bb95e0b963ab6fb19212eb441a9 Mon Sep 17 00:00:00 2001
From: Max Dymond 
Date: Fri, 28 Jun 2019 22:19:27 +0100
Subject: Write a simple decompress target as well

---
 ossfuzz/Makefile             |  7 ++++---
 ossfuzz/decompress_fuzzer.cc | 28 ++++++++++++++++++++++++++++
 ossfuzz/ossfuzz.sh           |  4 ++--
 3 files changed, 34 insertions(+), 5 deletions(-)
 create mode 100644 ossfuzz/decompress_fuzzer.cc

diff --git a/ossfuzz/Makefile b/ossfuzz/Makefile
index 2a7e439..1e7679b 100644
--- a/ossfuzz/Makefile
+++ b/ossfuzz/Makefile
@@ -21,7 +21,8 @@
 #  - LZ4 homepage : http://www.lz4.org
 #  - LZ4 source repository : https://github.com/lz4/lz4
 # ##########################################################################
-# lz4_fuzzer : OSS Fuzz test tool
+# compress_fuzzer : OSS Fuzz test tool
+# decompress_fuzzer : OSS Fuzz test tool
 # ##########################################################################
 
 LZ4DIR  := ../lib
@@ -44,8 +45,8 @@ $(LZ4DIR)/liblz4.a:
 %.o: %.cc
 	$(CXX) -c $(CXXFLAGS) $(CPPFLAGS) $< -o $@
 
-.PHONY: compress_fuzzer
-compress_fuzzer: compress_fuzzer.o $(LZ4DIR)/liblz4.a
+# Generic rule for generating fuzzers
+%_fuzzer: %_fuzzer.o $(LZ4DIR)/liblz4.a
 	# Compile the standalone code just in case. The OSS-Fuzz code might
 	# override the LIB_FUZZING_ENGINE value to "-fsanitize=fuzzer"
 	$(CXX) -c $(CXXFLAGS) $(CPPFLAGS) standaloneengine.cc -o standaloneengine.o
diff --git a/ossfuzz/decompress_fuzzer.cc b/ossfuzz/decompress_fuzzer.cc
new file mode 100644
index 0000000..594a5af
--- /dev/null
+++ b/ossfuzz/decompress_fuzzer.cc
@@ -0,0 +1,28 @@
+#include 
+#include 
+#include 
+#include "lz4.h"
+
+#define CHECK(COND)   if (!(COND)) { abort(); }
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
+{
+  size_t const buffer_size = 10 * 1024 * 1024;
+  char *const dest_buffer = (char *)malloc(buffer_size);
+
+  if (dest_buffer != NULL)
+  {
+    // Allocation succeeded, try decompressing the incoming data.
+    int result = LZ4_decompress_safe((const char*)data,
+                                     dest_buffer,
+                                     size,
+                                     buffer_size);
+
+    // Ignore the result of decompression.
+    (void)result;
+
+    free(dest_buffer);
+  }
+
+  return 0;
+}
diff --git a/ossfuzz/ossfuzz.sh b/ossfuzz/ossfuzz.sh
index 87bc213..a76b0d6 100755
--- a/ossfuzz/ossfuzz.sh
+++ b/ossfuzz/ossfuzz.sh
@@ -16,8 +16,8 @@ echo "OUT: $OUT"
 export MAKEFLAGS+="-j$(nproc)"
 
 pushd ossfuzz
-make V=1 compress_fuzzer
+make V=1 compress_fuzzer decompress_fuzzer
 popd
 
 # Copy the fuzzers to the target directory.
-cp -v ossfuzz/compress_fuzzer $OUT/
+cp -v ossfuzz/compress_fuzzer ossfuzz/decompress_fuzzer $OUT/
-- 
cgit v0.12


From e72d44230093f58be47c855e6b7d92493ce160db Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 28 Jun 2019 14:40:14 -0700
Subject: Fix out-of-bounds read of up to 64 KB in the past

---
 lib/lz4.c | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index cac3240..d121e29 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1703,6 +1703,7 @@ LZ4_decompress_generic(
             /* get offset */
             offset = LZ4_readLE16(ip); ip+=2;
             match = op - offset;
+            assert(match <= op);
 
             /* get matchlength */
             length = token & ML_MASK;
@@ -1724,8 +1725,12 @@ LZ4_decompress_generic(
                 }
 
                 /* Fastpath check: Avoids a branch in LZ4_wildCopy32 if true */
-                if (!(dict == usingExtDict) || (match >= lowPrefix)) {
+                if ((dict == withPrefix64k) || (match >= lowPrefix)) {
                     if (offset >= 8) {
+                        assert(match >= lowPrefix);
+                        assert(match <= op);
+                        assert(op + 18 <= oend);
+
                         memcpy(op, match, 8);
                         memcpy(op+8, match+8, 8);
                         memcpy(op+16, match+16, 2);
@@ -1873,7 +1878,6 @@ LZ4_decompress_generic(
             length = token & ML_MASK;
 
     _copy_match:
-            if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error;   /* Error : offset outside buffers */
             if (!partialDecoding) {
                 assert(oend > op);
                 assert(oend - op >= 4);
@@ -1891,6 +1895,7 @@ LZ4_decompress_generic(
 #if LZ4_FAST_DEC_LOOP
         safe_match_copy:
 #endif
+            if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error;   /* Error : offset outside buffers */
             /* match starting within external dictionary */
             if ((dict==usingExtDict) && (match < lowPrefix)) {
                 if (unlikely(op+length > oend-LASTLITERALS)) {
@@ -1918,6 +1923,7 @@ LZ4_decompress_generic(
                 }   }
                 continue;
             }
+            assert(match >= lowPrefix);
 
             /* copy match within block */
             cpy = op + length;
-- 
cgit v0.12


From 02b5b3c242fd4131983152f0dd422429e6702923 Mon Sep 17 00:00:00 2001
From: Max Dymond 
Date: Fri, 28 Jun 2019 23:48:33 +0100
Subject: Move to using C rather than C++ for compilation

---
 Makefile                     |  1 +
 ossfuzz/Makefile             | 12 +++++--
 ossfuzz/compress_fuzzer.c    | 26 ++++++++++++++++
 ossfuzz/compress_fuzzer.cc   | 26 ----------------
 ossfuzz/decompress_fuzzer.c  | 28 +++++++++++++++++
 ossfuzz/decompress_fuzzer.cc | 28 -----------------
 ossfuzz/standaloneengine.c   | 74 ++++++++++++++++++++++++++++++++++++++++++++
 ossfuzz/standaloneengine.cc  | 74 --------------------------------------------
 ossfuzz/testinput.h          |  2 +-
 9 files changed, 139 insertions(+), 132 deletions(-)
 create mode 100644 ossfuzz/compress_fuzzer.c
 delete mode 100644 ossfuzz/compress_fuzzer.cc
 create mode 100644 ossfuzz/decompress_fuzzer.c
 delete mode 100644 ossfuzz/decompress_fuzzer.cc
 create mode 100644 ossfuzz/standaloneengine.c
 delete mode 100644 ossfuzz/standaloneengine.cc

diff --git a/Makefile b/Makefile
index 34835fd..f25f951 100644
--- a/Makefile
+++ b/Makefile
@@ -77,6 +77,7 @@ clean:
 	@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
 	@$(MAKE) -C $(TESTDIR) $@ > $(VOID)
 	@$(MAKE) -C $(EXDIR) $@ > $(VOID)
+	@$(MAKE) -C $(FUZZDIR) $@ > $(VOID)
 	@$(MAKE) -C contrib/gen_manual $@ > $(VOID)
 	@$(RM) lz4$(EXT)
 	@echo Cleaning completed
diff --git a/ossfuzz/Makefile b/ossfuzz/Makefile
index 1e7679b..1480ccb 100644
--- a/ossfuzz/Makefile
+++ b/ossfuzz/Makefile
@@ -42,14 +42,20 @@ include ../Makefile.inc
 $(LZ4DIR)/liblz4.a:
 	$(MAKE) -C $(LZ4DIR) CFLAGS="$(CFLAGS)" liblz4.a
 
-%.o: %.cc
-	$(CXX) -c $(CXXFLAGS) $(CPPFLAGS) $< -o $@
+%.o: %.c
+	$(CC) -c $(CFLAGS) $(CPPFLAGS) $< -o $@
 
 # Generic rule for generating fuzzers
 %_fuzzer: %_fuzzer.o $(LZ4DIR)/liblz4.a
 	# Compile the standalone code just in case. The OSS-Fuzz code might
 	# override the LIB_FUZZING_ENGINE value to "-fsanitize=fuzzer"
-	$(CXX) -c $(CXXFLAGS) $(CPPFLAGS) standaloneengine.cc -o standaloneengine.o
+	$(CC) -c $(CFLAGS) $(CPPFLAGS) standaloneengine.c -o standaloneengine.o
 
 	# Now compile the actual fuzzer.
 	$(CXX) $(CXXFLAGS) $(CPPFLAGS) $(LDFLAGS) $(LIB_FUZZING_ENGINE) $^ -o $@$(EXT)
+
+%_fuzzer_clean:
+	$(RM) $*_fuzzer $*_fuzzer.o standaloneengine.o
+
+.PHONY: clean
+clean: compress_fuzzer_clean decompress_fuzzer_clean
diff --git a/ossfuzz/compress_fuzzer.c b/ossfuzz/compress_fuzzer.c
new file mode 100644
index 0000000..28610ad
--- /dev/null
+++ b/ossfuzz/compress_fuzzer.c
@@ -0,0 +1,26 @@
+#include 
+#include 
+#include 
+#include "lz4.h"
+
+#define CHECK(COND)   if (!(COND)) { abort(); }
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
+{
+  size_t const compressed_dest_size = LZ4_compressBound(size);
+  char *const dest_buffer = (char *)malloc(compressed_dest_size);
+
+  if (dest_buffer != NULL)
+  {
+    // Allocation succeeded, try compressing the incoming data.
+    int result = LZ4_compress_default((const char*)data,
+                                      dest_buffer,
+                                      size,
+                                      compressed_dest_size);
+    CHECK(result != 0);
+
+    free(dest_buffer);
+  }
+
+  return 0;
+}
diff --git a/ossfuzz/compress_fuzzer.cc b/ossfuzz/compress_fuzzer.cc
deleted file mode 100644
index 4a720e2..0000000
--- a/ossfuzz/compress_fuzzer.cc
+++ /dev/null
@@ -1,26 +0,0 @@
-#include 
-#include 
-#include 
-#include "lz4.h"
-
-#define CHECK(COND)   if (!(COND)) { abort(); }
-
-extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
-{
-  size_t const compressed_dest_size = LZ4_compressBound(size);
-  char *const dest_buffer = (char *)malloc(compressed_dest_size);
-
-  if (dest_buffer != NULL)
-  {
-    // Allocation succeeded, try compressing the incoming data.
-    int result = LZ4_compress_default((const char*)data,
-                                      dest_buffer,
-                                      size,
-                                      compressed_dest_size);
-    CHECK(result != 0);
-
-    free(dest_buffer);
-  }
-
-  return 0;
-}
diff --git a/ossfuzz/decompress_fuzzer.c b/ossfuzz/decompress_fuzzer.c
new file mode 100644
index 0000000..1fa2b1a
--- /dev/null
+++ b/ossfuzz/decompress_fuzzer.c
@@ -0,0 +1,28 @@
+#include 
+#include 
+#include 
+#include "lz4.h"
+
+#define CHECK(COND)   if (!(COND)) { abort(); }
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
+{
+  size_t const buffer_size = 10 * 1024 * 1024;
+  char *const dest_buffer = (char *)malloc(buffer_size);
+
+  if (dest_buffer != NULL)
+  {
+    // Allocation succeeded, try decompressing the incoming data.
+    int result = LZ4_decompress_safe((const char*)data,
+                                     dest_buffer,
+                                     size,
+                                     buffer_size);
+
+    // Ignore the result of decompression.
+    (void)result;
+
+    free(dest_buffer);
+  }
+
+  return 0;
+}
diff --git a/ossfuzz/decompress_fuzzer.cc b/ossfuzz/decompress_fuzzer.cc
deleted file mode 100644
index 594a5af..0000000
--- a/ossfuzz/decompress_fuzzer.cc
+++ /dev/null
@@ -1,28 +0,0 @@
-#include 
-#include 
-#include 
-#include "lz4.h"
-
-#define CHECK(COND)   if (!(COND)) { abort(); }
-
-extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
-{
-  size_t const buffer_size = 10 * 1024 * 1024;
-  char *const dest_buffer = (char *)malloc(buffer_size);
-
-  if (dest_buffer != NULL)
-  {
-    // Allocation succeeded, try decompressing the incoming data.
-    int result = LZ4_decompress_safe((const char*)data,
-                                     dest_buffer,
-                                     size,
-                                     buffer_size);
-
-    // Ignore the result of decompression.
-    (void)result;
-
-    free(dest_buffer);
-  }
-
-  return 0;
-}
diff --git a/ossfuzz/standaloneengine.c b/ossfuzz/standaloneengine.c
new file mode 100644
index 0000000..175360e
--- /dev/null
+++ b/ossfuzz/standaloneengine.c
@@ -0,0 +1,74 @@
+#include 
+#include 
+#include 
+
+#include "testinput.h"
+
+/**
+ * Main procedure for standalone fuzzing engine.
+ *
+ * Reads filenames from the argument array. For each filename, read the file
+ * into memory and then call the fuzzing interface with the data.
+ */
+int main(int argc, char **argv)
+{
+  int ii;
+  for(ii = 1; ii < argc; ii++)
+  {
+    FILE *infile;
+    printf("[%s] ", argv[ii]);
+
+    /* Try and open the file. */
+    infile = fopen(argv[ii], "rb");
+    if(infile)
+    {
+      uint8_t *buffer = NULL;
+      size_t buffer_len;
+
+      printf("Opened.. ");
+
+      /* Get the length of the file. */
+      fseek(infile, 0L, SEEK_END);
+      buffer_len = ftell(infile);
+
+      /* Reset the file indicator to the beginning of the file. */
+      fseek(infile, 0L, SEEK_SET);
+
+      /* Allocate a buffer for the file contents. */
+      buffer = (uint8_t *)calloc(buffer_len, sizeof(uint8_t));
+      if(buffer)
+      {
+        /* Read all the text from the file into the buffer. */
+        fread(buffer, sizeof(uint8_t), buffer_len, infile);
+        printf("Read %zu bytes, fuzzing.. ", buffer_len);
+
+        /* Call the fuzzer with the data. */
+        LLVMFuzzerTestOneInput(buffer, buffer_len);
+
+        printf("complete !!");
+
+        /* Free the buffer as it's no longer needed. */
+        free(buffer);
+        buffer = NULL;
+      }
+      else
+      {
+        fprintf(stderr,
+                "[%s] Failed to allocate %zu bytes \n",
+                argv[ii],
+                buffer_len);
+      }
+
+      /* Close the file as it's no longer needed. */
+      fclose(infile);
+      infile = NULL;
+    }
+    else
+    {
+      /* Failed to open the file. Maybe wrong name or wrong permissions? */
+      fprintf(stderr, "[%s] Open failed. \n", argv[ii]);
+    }
+
+    printf("\n");
+  }
+}
diff --git a/ossfuzz/standaloneengine.cc b/ossfuzz/standaloneengine.cc
deleted file mode 100644
index 175360e..0000000
--- a/ossfuzz/standaloneengine.cc
+++ /dev/null
@@ -1,74 +0,0 @@
-#include 
-#include 
-#include 
-
-#include "testinput.h"
-
-/**
- * Main procedure for standalone fuzzing engine.
- *
- * Reads filenames from the argument array. For each filename, read the file
- * into memory and then call the fuzzing interface with the data.
- */
-int main(int argc, char **argv)
-{
-  int ii;
-  for(ii = 1; ii < argc; ii++)
-  {
-    FILE *infile;
-    printf("[%s] ", argv[ii]);
-
-    /* Try and open the file. */
-    infile = fopen(argv[ii], "rb");
-    if(infile)
-    {
-      uint8_t *buffer = NULL;
-      size_t buffer_len;
-
-      printf("Opened.. ");
-
-      /* Get the length of the file. */
-      fseek(infile, 0L, SEEK_END);
-      buffer_len = ftell(infile);
-
-      /* Reset the file indicator to the beginning of the file. */
-      fseek(infile, 0L, SEEK_SET);
-
-      /* Allocate a buffer for the file contents. */
-      buffer = (uint8_t *)calloc(buffer_len, sizeof(uint8_t));
-      if(buffer)
-      {
-        /* Read all the text from the file into the buffer. */
-        fread(buffer, sizeof(uint8_t), buffer_len, infile);
-        printf("Read %zu bytes, fuzzing.. ", buffer_len);
-
-        /* Call the fuzzer with the data. */
-        LLVMFuzzerTestOneInput(buffer, buffer_len);
-
-        printf("complete !!");
-
-        /* Free the buffer as it's no longer needed. */
-        free(buffer);
-        buffer = NULL;
-      }
-      else
-      {
-        fprintf(stderr,
-                "[%s] Failed to allocate %zu bytes \n",
-                argv[ii],
-                buffer_len);
-      }
-
-      /* Close the file as it's no longer needed. */
-      fclose(infile);
-      infile = NULL;
-    }
-    else
-    {
-      /* Failed to open the file. Maybe wrong name or wrong permissions? */
-      fprintf(stderr, "[%s] Open failed. \n", argv[ii]);
-    }
-
-    printf("\n");
-  }
-}
diff --git a/ossfuzz/testinput.h b/ossfuzz/testinput.h
index 6ab9b51..8da6215 100644
--- a/ossfuzz/testinput.h
+++ b/ossfuzz/testinput.h
@@ -1,3 +1,3 @@
 #include 
 
-extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size);
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size);
-- 
cgit v0.12


From e2a33f12e1198d3f5de374519d7a034a8736f6c8 Mon Sep 17 00:00:00 2001
From: Max Dymond 
Date: Sat, 29 Jun 2019 00:23:06 +0100
Subject: More markups for style changes

---
 ossfuzz/Makefile            | 14 +++++++-------
 ossfuzz/compress_fuzzer.c   | 19 +++++++++----------
 ossfuzz/decompress_fuzzer.c | 22 +++++++++++-----------
 ossfuzz/testinput.h         | 12 ++++++++++++
 4 files changed, 39 insertions(+), 28 deletions(-)

diff --git a/ossfuzz/Makefile b/ossfuzz/Makefile
index 1480ccb..7812c41 100644
--- a/ossfuzz/Makefile
+++ b/ossfuzz/Makefile
@@ -31,28 +31,28 @@ LIB_FUZZING_ENGINE ?= standaloneengine.o
 DEBUGLEVEL?= 1
 DEBUGFLAGS = -g -DLZ4_DEBUG=$(DEBUGLEVEL)
 
-CFLAGS   += -I$(LZ4DIR) $(DEBUGFLAGS) $(MOREFLAGS)
-CXXFLAGS += -I$(LZ4DIR) $(DEBUGFLAGS) $(MOREFLAGS)
-CPPFLAGS += -DXXH_NAMESPACE=LZ4_
+LZ4_CFLAGS  = $(CFLAGS) $(DEBUGFLAGS) $(MOREFLAGS)
+LZ4_CXXFLAGS = $(CXXFLAGS) $(DEBUGFLAGS) $(MOREFLAGS)
+LZ4_CPPFLAGS = $(CPPFLAGS) -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_
 
 include ../Makefile.inc
 
 # Include a rule to build the static library if calling this target
 # directly.
 $(LZ4DIR)/liblz4.a:
-	$(MAKE) -C $(LZ4DIR) CFLAGS="$(CFLAGS)" liblz4.a
+	$(MAKE) -C $(LZ4DIR) CFLAGS="$(LZ4_CFLAGS)" liblz4.a
 
 %.o: %.c
-	$(CC) -c $(CFLAGS) $(CPPFLAGS) $< -o $@
+	$(CC) -c $(LZ4_CFLAGS) $(LZ4_CPPFLAGS) $< -o $@
 
 # Generic rule for generating fuzzers
 %_fuzzer: %_fuzzer.o $(LZ4DIR)/liblz4.a
 	# Compile the standalone code just in case. The OSS-Fuzz code might
 	# override the LIB_FUZZING_ENGINE value to "-fsanitize=fuzzer"
-	$(CC) -c $(CFLAGS) $(CPPFLAGS) standaloneengine.c -o standaloneengine.o
+	$(CC) -c $(LZ4_CFLAGS) $(LZ4_CPPFLAGS) standaloneengine.c -o standaloneengine.o
 
 	# Now compile the actual fuzzer.
-	$(CXX) $(CXXFLAGS) $(CPPFLAGS) $(LDFLAGS) $(LIB_FUZZING_ENGINE) $^ -o $@$(EXT)
+	$(CXX) $(LZ4_CXXFLAGS) $(LZ4_CPPFLAGS) $(LDFLAGS) $(LIB_FUZZING_ENGINE) $^ -o $@$(EXT)
 
 %_fuzzer_clean:
 	$(RM) $*_fuzzer $*_fuzzer.o standaloneengine.o
diff --git a/ossfuzz/compress_fuzzer.c b/ossfuzz/compress_fuzzer.c
index 28610ad..3908534 100644
--- a/ossfuzz/compress_fuzzer.c
+++ b/ossfuzz/compress_fuzzer.c
@@ -10,17 +10,16 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
   size_t const compressed_dest_size = LZ4_compressBound(size);
   char *const dest_buffer = (char *)malloc(compressed_dest_size);
 
-  if (dest_buffer != NULL)
-  {
-    // Allocation succeeded, try compressing the incoming data.
-    int result = LZ4_compress_default((const char*)data,
-                                      dest_buffer,
-                                      size,
-                                      compressed_dest_size);
-    CHECK(result != 0);
+  CHECK(dest_buffer != NULL);
 
-    free(dest_buffer);
-  }
+  // Allocation succeeded, try compressing the incoming data.
+  int result = LZ4_compress_default((const char*)data,
+                                    dest_buffer,
+                                    size,
+                                    compressed_dest_size);
+  CHECK(result != 0);
+
+  free(dest_buffer);
 
   return 0;
 }
diff --git a/ossfuzz/decompress_fuzzer.c b/ossfuzz/decompress_fuzzer.c
index 1fa2b1a..e6e14c4 100644
--- a/ossfuzz/decompress_fuzzer.c
+++ b/ossfuzz/decompress_fuzzer.c
@@ -7,22 +7,22 @@
 
 int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 {
+  // TODO: Size input buffer pseudo-randomly based on seed extracted from input
   size_t const buffer_size = 10 * 1024 * 1024;
   char *const dest_buffer = (char *)malloc(buffer_size);
 
-  if (dest_buffer != NULL)
-  {
-    // Allocation succeeded, try decompressing the incoming data.
-    int result = LZ4_decompress_safe((const char*)data,
-                                     dest_buffer,
-                                     size,
-                                     buffer_size);
+  CHECK(dest_buffer != NULL);
 
-    // Ignore the result of decompression.
-    (void)result;
+  // Allocation succeeded, try decompressing the incoming data.
+  int result = LZ4_decompress_safe((const char*)data,
+                                   dest_buffer,
+                                   size,
+                                   buffer_size);
 
-    free(dest_buffer);
-  }
+  // Ignore the result of decompression.
+  (void)result;
+
+  free(dest_buffer);
 
   return 0;
 }
diff --git a/ossfuzz/testinput.h b/ossfuzz/testinput.h
index 8da6215..0e50a3c 100644
--- a/ossfuzz/testinput.h
+++ b/ossfuzz/testinput.h
@@ -1,3 +1,15 @@
+#ifndef TESTINPUT_H_INCLUDED
+#define TESTINPUT_H_INCLUDED
+
 #include 
 
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
 int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size);
+
+#if defined(__cplusplus)
+}
+#endif
+#endif
-- 
cgit v0.12


From 4c60f25c6510f81a6bc2921a3fb7a77dc71f5d8a Mon Sep 17 00:00:00 2001
From: Lzu Tao 
Date: Sat, 29 Jun 2019 10:10:05 +0700
Subject: meson: Fix deprecated warnings on build options

Meson now reserves `build_` prefix options.
---
 contrib/meson/README.md         |  2 +-
 contrib/meson/meson.build       | 18 +++++++++---------
 contrib/meson/meson_options.txt |  8 ++++----
 3 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/contrib/meson/README.md b/contrib/meson/README.md
index fa18493..a44850a 100644
--- a/contrib/meson/README.md
+++ b/contrib/meson/README.md
@@ -13,7 +13,7 @@ This Meson build system is provided with no guarantee.
 `cd` to this meson directory (`contrib/meson`)
 
 ```sh
-meson setup --buildtype=release -Ddefault_library=shared -Dbuild_programs=true builddir
+meson setup --buildtype=release -Ddefault_library=shared -Dbin_programs=true builddir
 cd builddir
 ninja             # to build
 ninja install     # to install
diff --git a/contrib/meson/meson.build b/contrib/meson/meson.build
index c28d90a..65a4c26 100644
--- a/contrib/meson/meson.build
+++ b/contrib/meson/meson.build
@@ -69,17 +69,17 @@ use_debug = get_option('debug')
 debug_level = get_option('debug_level')
 use_backtrace = get_option('backtrace')
 
-build_programs = get_option('build_programs')
-build_contrib = get_option('build_contrib')
-build_tests = get_option('build_tests')
-build_examples = get_option('build_examples')
+bin_programs = get_option('bin_programs')
+bin_contrib = get_option('bin_contrib')
+bin_tests = get_option('bin_tests')
+bin_examples = get_option('bin_examples')
 #feature_multi_thread = get_option('multi_thread')
 
 # =============================================================================
 # Dependencies
 # =============================================================================
 
-#libm_dep = cc.find_library('m', required: build_tests)
+#libm_dep = cc.find_library('m', required: bin_tests)
 #thread_dep = dependency('threads', required: feature_multi_thread)
 #use_multi_thread = thread_dep.found()
 
@@ -108,18 +108,18 @@ endif
 
 subdir('lib')
 
-if build_programs
+if bin_programs
   subdir('programs')
 endif
 
-if build_tests
+if bin_tests
   subdir('tests')
 endif
 
-if build_contrib
+if bin_contrib
   subdir('contrib')
 endif
 
-if build_examples
+if bin_examples
   subdir('examples')
 endif
diff --git a/contrib/meson/meson_options.txt b/contrib/meson/meson_options.txt
index f6a4ae7..a409c2d 100644
--- a/contrib/meson/meson_options.txt
+++ b/contrib/meson/meson_options.txt
@@ -14,11 +14,11 @@ option('debug_level', type: 'integer', min: 0, max: 7, value: 1,
 option('backtrace', type: 'boolean', value: false,
   description: 'Display a stack backtrace when execution generates a runtime exception')
 
-option('build_programs', type: 'boolean', value: false,
+option('bin_programs', type: 'boolean', value: false,
   description: 'Enable programs build')
-option('build_tests', type: 'boolean', value: false,
+option('bin_tests', type: 'boolean', value: false,
   description: 'Enable tests build')
-option('build_contrib', type: 'boolean', value: false,
+option('bin_contrib', type: 'boolean', value: false,
   description: 'Enable contrib build')
-option('build_examples', type: 'boolean', value: false,
+option('bin_examples', type: 'boolean', value: false,
   description: 'Enable examples build')
-- 
cgit v0.12


From ff27a1572b830a82917eefc4008debc433f837d3 Mon Sep 17 00:00:00 2001
From: Lzu Tao 
Date: Sat, 29 Jun 2019 10:11:32 +0700
Subject: meson: Always build gen_manual on build machine

As gen_manual is using as a generator, not a binary target
installed on host machine.
---
 contrib/meson/contrib/gen_manual/meson.build | 1 +
 1 file changed, 1 insertion(+)

diff --git a/contrib/meson/contrib/gen_manual/meson.build b/contrib/meson/contrib/gen_manual/meson.build
index 6233cdc..38180e9 100644
--- a/contrib/meson/contrib/gen_manual/meson.build
+++ b/contrib/meson/contrib/gen_manual/meson.build
@@ -22,6 +22,7 @@ gen_manual = executable('gen_manual',
   join_paths(lz4_root_dir, 'contrib/gen_manual/gen_manual.cpp'),
   cpp_args: gen_manual_cppflags,
   include_directories: gen_manual_includes,
+  native: true,
   install: false)
 
 # Update lz4 manual
-- 
cgit v0.12


From 62f59d562b54e8329f585136d37d673f04933a19 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 28 Jun 2019 20:15:43 -0700
Subject: fuzzer: added test to catch #738

---
 tests/fuzzer.c | 141 ++++++++++++++++++++++++++++++++++++---------------------
 1 file changed, 90 insertions(+), 51 deletions(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index b45620b..db26b72 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -122,6 +122,14 @@ static U32 FUZ_rotl32(U32 u32, U32 nbBits)
     return ((u32 << nbBits) | (u32 >> (32 - nbBits)));
 }
 
+static U32 FUZ_highbit32(U32 v32)
+{
+    unsigned nbBits = 0;
+    if (v32==0) return 0;
+    while (v32) v32 >>= 1, nbBits++;
+    return nbBits;
+}
+
 static U32 FUZ_rand(U32* src)
 {
     U32 rand32 = *src;
@@ -366,7 +374,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         U32 testNb = 0;
         U32 randState = FUZ_rand(&coreRandState) ^ PRIME3;
         int const blockSize  = (FUZ_rand(&randState) % (FUZ_MAX_BLOCK_SIZE-1)) + 1;
-        int const blockStart = (int)(FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize - 1)) + 1;
+        int const blockStart = (int)(FUZ_rand(&randState) % (U32)(COMPRESSIBLE_NOISE_LENGTH - blockSize - 1)) + 1;
         int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
         int const dictSize = MIN(dictSizeRand, blockStart - 1);
         int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1);
@@ -388,69 +396,65 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
 
         /* Test compression destSize */
         FUZ_DISPLAYTEST("test LZ4_compress_destSize()");
-        {   int srcSize = blockSize;
+        {   int cSize, srcSize = blockSize;
             int const targetSize = srcSize * (int)((FUZ_rand(&randState) & 127)+1) >> 7;
-            char endCheck = (char)(FUZ_rand(&randState) & 255);
+            char const endCheck = (char)(FUZ_rand(&randState) & 255);
             compressedBuffer[targetSize] = endCheck;
-            ret = LZ4_compress_destSize(block, compressedBuffer, &srcSize, targetSize);
-            FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_destSize() result larger than dst buffer !");
+            cSize = LZ4_compress_destSize(block, compressedBuffer, &srcSize, targetSize);
+            FUZ_CHECKTEST(cSize > targetSize, "LZ4_compress_destSize() result larger than dst buffer !");
             FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_destSize() overwrite dst buffer !");
-            FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_destSize() fed more than src buffer !");
-            DISPLAYLEVEL(5, "destSize : %7i/%7i; content%7i/%7i ", ret, targetSize, srcSize, blockSize);
+            FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_destSize() read more than src buffer !");
+            DISPLAYLEVEL(5, "destSize : %7i/%7i; content%7i/%7i ", cSize, targetSize, srcSize, blockSize);
             if (targetSize>0) {
                 /* check correctness */
                 U32 const crcBase = XXH32(block, (size_t)srcSize, 0);
                 char const canary = (char)(FUZ_rand(&randState) & 255);
-                FUZ_CHECKTEST((ret==0), "LZ4_compress_destSize() compression failed");
+                FUZ_CHECKTEST((cSize==0), "LZ4_compress_destSize() compression failed");
                 FUZ_DISPLAYTEST();
-                compressedSize = ret;
                 decodedBuffer[srcSize] = canary;
-                ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize);
-                FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compress_destSize");
-                FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data");
+                {   int const dSize = LZ4_decompress_safe(compressedBuffer, decodedBuffer, cSize, srcSize);
+                    FUZ_CHECKTEST(dSize<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compress_destSize");
+                    FUZ_CHECKTEST(dSize!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data");
+                }
                 FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe() overwrite dst buffer !");
-                { U32 const crcDec = XXH32(decodedBuffer, (size_t)srcSize, 0);
-                  FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); }
-
-                DISPLAYLEVEL(5, " OK \n");
-            } else {
-                DISPLAYLEVEL(5, " \n");
-        }   }
+                {   U32 const crcDec = XXH32(decodedBuffer, (size_t)srcSize, 0);
+                    FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data");
+            }   }
+            DISPLAYLEVEL(5, " OK \n");
+        }
 
         /* Test compression HC destSize */
         FUZ_DISPLAYTEST("test LZ4_compress_HC_destSize()");
-        {   int srcSize = blockSize;
+        {   int cSize, srcSize = blockSize;
             int const targetSize = srcSize * (int)((FUZ_rand(&randState) & 127)+1) >> 7;
             char const endCheck = (char)(FUZ_rand(&randState) & 255);
-            void* ctx = LZ4_createHC(block);
+            void* const ctx = LZ4_createHC(block);
             FUZ_CHECKTEST(ctx==NULL, "LZ4_createHC() allocation failed");
             compressedBuffer[targetSize] = endCheck;
-            ret = LZ4_compress_HC_destSize(ctx, block, compressedBuffer, &srcSize, targetSize, compressionLevel);
+            cSize = LZ4_compress_HC_destSize(ctx, block, compressedBuffer, &srcSize, targetSize, compressionLevel);
             DISPLAYLEVEL(5, "LZ4_compress_HC_destSize(%i): destSize : %7i/%7i; content%7i/%7i ",
-                            compressionLevel, ret, targetSize, srcSize, blockSize);
+                            compressionLevel, cSize, targetSize, srcSize, blockSize);
             LZ4_freeHC(ctx);
-            FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_HC_destSize() result larger than dst buffer !");
+            FUZ_CHECKTEST(cSize > targetSize, "LZ4_compress_HC_destSize() result larger than dst buffer !");
             FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_HC_destSize() overwrite dst buffer !");
             FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_HC_destSize() fed more than src buffer !");
             if (targetSize>0) {
                 /* check correctness */
                 U32 const crcBase = XXH32(block, (size_t)srcSize, 0);
                 char const canary = (char)(FUZ_rand(&randState) & 255);
-                FUZ_CHECKTEST((ret==0), "LZ4_compress_HC_destSize() compression failed");
+                FUZ_CHECKTEST((cSize==0), "LZ4_compress_HC_destSize() compression failed");
                 FUZ_DISPLAYTEST();
-                compressedSize = ret;
                 decodedBuffer[srcSize] = canary;
-                ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize);
-                FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compressHC_destSize");
-                FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data");
+                {   int const dSize = LZ4_decompress_safe(compressedBuffer, decodedBuffer, cSize, srcSize);
+                    FUZ_CHECKTEST(dSize<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compressHC_destSize");
+                    FUZ_CHECKTEST(dSize!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data");
+                }
                 FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe() overwrite dst buffer !");
                 {   U32 const crcDec = XXH32(decodedBuffer, (size_t)srcSize, 0);
                     FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data");
-                }
-                DISPLAYLEVEL(5, " OK \n");
-            } else {
-                DISPLAYLEVEL(5, " \n");
-        }   }
+            }   }
+            DISPLAYLEVEL(5, " OK \n");
+        }
 
         /* Test compression HC */
         FUZ_DISPLAYTEST("test LZ4_compress_HC()");
@@ -565,7 +569,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         /* Test decoding with input size being one byte too short => must fail */
         FUZ_DISPLAYTEST();
         {   int const r = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize);
-            FUZ_CHECKTEST(r>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize);
+            FUZ_CHECKTEST(r>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, result=%i, compressedSize=%i)", blockSize, r, compressedSize);
         }
 
         /* Test decoding with input size being one byte too large => must fail */
@@ -587,6 +591,42 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_CHECKTEST(decodedBuffer[targetSize] != sentinel, "LZ4_decompress_safe_partial overwrite beyond requested size (though %i <= %i <= %i)", decResult, targetSize, blockSize);
         }
 
+        /* noisy src decompression test */
+        {
+            /* insert noise into src */
+            {   U32 const maxNbBits = FUZ_highbit32((U32)compressedSize);
+                size_t pos = 0;
+                for (;;) {
+                    /* keep some original src */
+                    {   U32 const nbBits = FUZ_rand(&randState) % maxNbBits;
+                        size_t const mask = (1<= (size_t)compressedSize) break;
+                    /* add noise */
+                    {   U32 const nbBitsCodes = FUZ_rand(&randState) % maxNbBits;
+                        U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0;
+                        size_t const mask = (1< blockSize, "LZ4_decompress_safe on noisy src : result is too large : %u > %u (dst buffer)", (unsigned)decompressResult, (unsigned)blockSize);
+                }
+                {   U32 endCheck; memcpy(&endCheck, decodedBuffer+blockSize, sizeof(endCheck));
+                    FUZ_CHECKTEST(endMark!=endCheck, "LZ4_decompress_safe on noisy src : dst buffer overflow");
+        }   }   }   /* noisy src decompression test */
+
         /* Test Compression with limited output size */
 
         /* Test compression with output size being exactly what's necessary (should work) */
@@ -615,8 +655,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             if (missingBytes >= compressedSize) missingBytes = compressedSize-1;
             missingBytes += !missingBytes;   /* avoid special case missingBytes==0 */
             compressedBuffer[compressedSize-missingBytes] = 0;
-            ret = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize-missingBytes);
-            FUZ_CHECKTEST(ret, "LZ4_compress_default should have failed (output buffer too small by %i byte)", missingBytes);
+            {   int const cSize = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize-missingBytes);
+                FUZ_CHECKTEST(cSize, "LZ4_compress_default should have failed (output buffer too small by %i byte)", missingBytes);
+            }
             FUZ_CHECKTEST(compressedBuffer[compressedSize-missingBytes], "LZ4_compress_default overran output buffer ! (%i missingBytes)", missingBytes)
         }
 
@@ -626,8 +667,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             if (missingBytes >= HCcompressedSize) missingBytes = HCcompressedSize-1;
             missingBytes += !missingBytes;   /* avoid special case missingBytes==0 */
             compressedBuffer[HCcompressedSize-missingBytes] = 0;
-            ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, compressionLevel);
-            FUZ_CHECKTEST(ret, "LZ4_compress_HC should have failed (output buffer too small by %i byte)", missingBytes);
+            {   int const hcSize = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, compressionLevel);
+                FUZ_CHECKTEST(hcSize, "LZ4_compress_HC should have failed (output buffer too small by %i byte)", missingBytes);
+            }
             FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-missingBytes], "LZ4_compress_HC overran output buffer ! (%i missingBytes)", missingBytes)
         }
 
@@ -654,8 +696,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             if (crcCheck!=crcOrig) {
                 FUZ_findDiff(block, decodedBuffer);
                 EXIT_MSG("LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
-            }
-        }
+        }   }
 
         FUZ_DISPLAYTEST("test LZ4_decompress_safe_usingDict()");
         ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize);
@@ -697,8 +738,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             if (crcCheck!=crcOrig) {
                 FUZ_findDiff(block, decodedBuffer);
                 EXIT_MSG("LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
-            }
-        }
+        }   }
 
         FUZ_DISPLAYTEST();
         decodedBuffer[blockSize] = 0;
@@ -722,8 +762,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
 
         FUZ_DISPLAYTEST();
-        {   U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
-            if ((U32)blockSize > missingBytes) {
+        {   int const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
+            if (blockSize > missingBytes) {
                 decodedBuffer[blockSize-missingBytes] = 0;
                 ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-missingBytes, dict, dictSize);
                 FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%u byte)", missingBytes);
@@ -731,8 +771,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }   }
 
         /* Compress using external dictionary stream */
-        {
-            LZ4_stream_t LZ4_stream;
+        {   LZ4_stream_t LZ4_stream;
             int expectedSize;
             U32 expectedCrc;
 
@@ -740,7 +779,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             LZ4_loadDict(&LZ4dictBody, dict, dictSize);
             expectedSize = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
             FUZ_CHECKTEST(expectedSize<=0, "LZ4_compress_fast_continue reference compression for extDictCtx should have succeeded");
-            expectedCrc = XXH32(compressedBuffer, expectedSize, 0);
+            expectedCrc = XXH32(compressedBuffer, (size_t)expectedSize, 0);
 
             FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary()");
             LZ4_loadDict(&LZ4dictBody, dict, dictSize);
@@ -756,7 +795,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
              * test.
              */
             FUZ_CHECKTEST(blockContinueCompressedSize != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output (%d expected vs %d actual)", expectedSize, blockContinueCompressedSize);
-            FUZ_CHECKTEST(XXH32(compressedBuffer, blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
+            FUZ_CHECKTEST(XXH32(compressedBuffer, (size_t)blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
 
             FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary(), but output buffer is 1 byte too short");
             LZ4_resetStream_fast(&LZ4_stream);
@@ -772,7 +811,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
             FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx should work : enough size available within output buffer");
             FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output");
-            FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
+            FUZ_CHECKTEST(XXH32(compressedBuffer, (size_t)ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
             FUZ_CHECKTEST(LZ4_stream.internal_donotuse.dirty, "context should be good");
 
             FUZ_DISPLAYTEST();
@@ -782,7 +821,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
             FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx with re-used context should work : enough size available within output buffer");
             FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output");
-            FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
+            FUZ_CHECKTEST(XXH32(compressedBuffer, (size_t)ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
             FUZ_CHECKTEST(LZ4_stream.internal_donotuse.dirty, "context should be good");
         }
 
-- 
cgit v0.12


From 567b4e098baad783870bf6514a984c4b021d5056 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 28 Jun 2019 20:23:12 -0700
Subject: moved noisy-src decoder test into cBuffer_exact

so that it can also catch any potential read out-of-bound in the input buffer
(none reported so far, just a precaution for the future).
---
 tests/fuzzer.c | 68 +++++++++++++++++++++++++++++-----------------------------
 1 file changed, 34 insertions(+), 34 deletions(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index db26b72..368b28b 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -563,36 +563,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
                 FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe overrun specified output buffer size");
             }
 
-            free(cBuffer_exact);
-        }
-
-        /* Test decoding with input size being one byte too short => must fail */
-        FUZ_DISPLAYTEST();
-        {   int const r = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize);
-            FUZ_CHECKTEST(r>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, result=%i, compressedSize=%i)", blockSize, r, compressedSize);
-        }
-
-        /* Test decoding with input size being one byte too large => must fail */
-        FUZ_DISPLAYTEST();
-        decodedBuffer[blockSize] = 0;
-        {   int const r = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize+1, blockSize);
-            FUZ_CHECKTEST(r>=0, "LZ4_decompress_safe should have failed, due to input size being too large");
-        }
-        FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
-
-        /* Test partial decoding => must work */
-        FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial");
-        {   size_t const missingBytes = FUZ_rand(&randState) % (unsigned)blockSize;
-            int const targetSize = (int)((size_t)blockSize - missingBytes);
-            char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A;
-            int const decResult = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, targetSize, blockSize);
-            FUZ_CHECKTEST(decResult<0, "LZ4_decompress_safe_partial failed despite valid input data (error:%i)", decResult);
-            FUZ_CHECKTEST(decResult != targetSize, "LZ4_decompress_safe_partial did not regenerated required amount of data (%i < %i <= %i)", decResult, targetSize, blockSize);
-            FUZ_CHECKTEST(decodedBuffer[targetSize] != sentinel, "LZ4_decompress_safe_partial overwrite beyond requested size (though %i <= %i <= %i)", decResult, targetSize, blockSize);
-        }
+            /* noisy src decompression test */
 
-        /* noisy src decompression test */
-        {
             /* insert noise into src */
             {   U32 const maxNbBits = FUZ_highbit32((U32)compressedSize);
                 size_t pos = 0;
@@ -611,21 +583,49 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
                         size_t const rNoiseLength = (FUZ_rand(&randState) & mask) + 1;
                         size_t const noiseLength = MIN(rNoiseLength, (size_t)compressedSize-pos);
                         size_t const noiseStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - noiseLength);
-                        memcpy(compressedBuffer + pos, (const char*)CNBuffer + noiseStart, noiseLength);
+                        memcpy(cBuffer_exact + pos, (const char*)CNBuffer + noiseStart, noiseLength);
                         pos += noiseLength;
             }   }   }
 
             /* decompress noisy source */
-            FUZ_DISPLAYTEST("decompress noisy source \n");
+            FUZ_DISPLAYTEST("decompress noisy source ");
             {   U32 const endMark = 0xA9B1C3D6;
                 memcpy(decodedBuffer+blockSize, &endMark, sizeof(endMark));
-                {   int const decompressResult = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize);
+                {   int const decompressResult = LZ4_decompress_safe(cBuffer_exact, decodedBuffer, compressedSize, blockSize);
                     /* result *may* be an unlikely success, but even then, it must strictly respect dst buffer boundaries */
                     FUZ_CHECKTEST(decompressResult > blockSize, "LZ4_decompress_safe on noisy src : result is too large : %u > %u (dst buffer)", (unsigned)decompressResult, (unsigned)blockSize);
                 }
                 {   U32 endCheck; memcpy(&endCheck, decodedBuffer+blockSize, sizeof(endCheck));
                     FUZ_CHECKTEST(endMark!=endCheck, "LZ4_decompress_safe on noisy src : dst buffer overflow");
-        }   }   }   /* noisy src decompression test */
+            }   }   /* noisy src decompression test */
+
+            free(cBuffer_exact);
+        }
+
+        /* Test decoding with input size being one byte too short => must fail */
+        FUZ_DISPLAYTEST();
+        {   int const r = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize);
+            FUZ_CHECKTEST(r>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, result=%i, compressedSize=%i)", blockSize, r, compressedSize);
+        }
+
+        /* Test decoding with input size being one byte too large => must fail */
+        FUZ_DISPLAYTEST();
+        decodedBuffer[blockSize] = 0;
+        {   int const r = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize+1, blockSize);
+            FUZ_CHECKTEST(r>=0, "LZ4_decompress_safe should have failed, due to input size being too large");
+        }
+        FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
+
+        /* Test partial decoding => must work */
+        FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial");
+        {   size_t const missingBytes = FUZ_rand(&randState) % (unsigned)blockSize;
+            int const targetSize = (int)((size_t)blockSize - missingBytes);
+            char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A;
+            int const decResult = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, targetSize, blockSize);
+            FUZ_CHECKTEST(decResult<0, "LZ4_decompress_safe_partial failed despite valid input data (error:%i)", decResult);
+            FUZ_CHECKTEST(decResult != targetSize, "LZ4_decompress_safe_partial did not regenerated required amount of data (%i < %i <= %i)", decResult, targetSize, blockSize);
+            FUZ_CHECKTEST(decodedBuffer[targetSize] != sentinel, "LZ4_decompress_safe_partial overwrite beyond requested size (though %i <= %i <= %i)", decResult, targetSize, blockSize);
+        }
 
         /* Test Compression with limited output size */
 
@@ -713,7 +713,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
         FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue failed");
 
-        FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary but with an output buffer too short by one byte");
+        FUZ_DISPLAYTEST("LZ4_compress_fast_continue() with dictionary and output buffer too short by one byte");
         LZ4_loadDict(&LZ4dictBody, dict, dictSize);
         ret = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
         FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
-- 
cgit v0.12


From 3b917ef6e6e9b18b15f18b6d691c4ea5033cfe41 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 28 Jun 2019 20:55:47 -0700
Subject: travisCI: added ASAN fuzzer tests

and fixed minor formatting warnings
---
 .travis.yml                       |  6 ++++++
 examples/HCStreaming_ringBuffer.c |  9 ++++++---
 tests/fullbench.c                 | 16 +++++++++-------
 tests/fuzzer.c                    |  6 +++---
 4 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index ee643e5..b88d907 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -27,6 +27,12 @@ matrix:
       script:
         - make -C tests test-frametest test-fuzzer
 
+    - name: ASAN tests with fuzzer and frametest
+      install:
+        - sudo sysctl -w vm.mmap_min_addr=4096
+      script:
+        - CC=clang MOREFLAGS=-fsanitize=address make -C tests test-frametest test-fuzzer
+
     - name: (Precise) g++ and clang CMake test
       dist: precise
       script:
diff --git a/examples/HCStreaming_ringBuffer.c b/examples/HCStreaming_ringBuffer.c
index a878577..bc8391e 100644
--- a/examples/HCStreaming_ringBuffer.c
+++ b/examples/HCStreaming_ringBuffer.c
@@ -26,6 +26,7 @@
 #include 
 #include 
 #include 
+#include 
 
 enum {
     MESSAGE_MAX_BYTES   = 1024,
@@ -39,7 +40,8 @@ size_t write_int32(FILE* fp, int32_t i) {
 }
 
 size_t write_bin(FILE* fp, const void* array, int arrayBytes) {
-    return fwrite(array, 1, arrayBytes, fp);
+    assert(arrayBytes >= 0);
+    return fwrite(array, 1, (size_t)arrayBytes, fp);
 }
 
 size_t read_int32(FILE* fp, int32_t* i) {
@@ -47,7 +49,8 @@ size_t read_int32(FILE* fp, int32_t* i) {
 }
 
 size_t read_bin(FILE* fp, void* array, int arrayBytes) {
-    return fread(array, 1, arrayBytes, fp);
+    assert(arrayBytes >= 0);
+    return fread(array, 1, (size_t)arrayBytes, fp);
 }
 
 
@@ -174,7 +177,7 @@ int main(int argc, const char** argv)
         return 0;
     }
 
-    if (!strcmp(argv[1], "-p")) pause = 1, fileID = 2;
+    if (!strcmp(argv[1], "-p")) { pause = 1; fileID = 2; }
 
     snprintf(inpFilename, 256, "%s", argv[fileID]);
     snprintf(lz4Filename, 256, "%s.lz4s-%d", argv[fileID], 9);
diff --git a/tests/fullbench.c b/tests/fullbench.c
index 4609f13..7d74d3f 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -541,9 +541,10 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
                     if (initFunction!=NULL) initFunction();
                     for (chunkNb=0; chunkNb%9i (%5.2f%%),%7.1f MB/s\r", loopNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000000);
             }
@@ -586,9 +587,10 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
         }
         for (chunkNb=0; chunkNb>= 1, nbBits++;
+    while (v32) { v32 >>= 1; nbBits++; }
     return nbBits;
 }
 
@@ -766,8 +766,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             if (blockSize > missingBytes) {
                 decodedBuffer[blockSize-missingBytes] = 0;
                 ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-missingBytes, dict, dictSize);
-                FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%u byte)", missingBytes);
-                FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize);
+                FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%i byte)", missingBytes);
+                FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%i byte) (blockSize=%i)", missingBytes, blockSize);
         }   }
 
         /* Compress using external dictionary stream */
-- 
cgit v0.12


From 84f98dd316e23fef5850645a0d7fc69ccff4c773 Mon Sep 17 00:00:00 2001
From: Lzu Tao 
Date: Sat, 29 Jun 2019 20:54:39 +0700
Subject: meson: Rename options in travis config

---
 .travis.yml | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index ee643e5..ca5fff3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -178,18 +178,24 @@ matrix:
       compiler: clang
       install:
         - sudo apt-get install -qq python3 tree
-        - travis_retry curl -o ~/ninja.zip -L 'https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-linux.zip'
-            && unzip ~/ninja.zip -d ~/.local/bin
-        - travis_retry curl -o ~/get-pip.py 'https://bootstrap.pypa.io/get-pip.py'
-            && python3 ~/get-pip.py --user
-            && pip3 install --user meson
-      script:
-        - meson setup
-            --buildtype=debug
-            -Db_lundef=false
-            -Dauto_features=enabled
-            -Ddefault_library=both
-            -Dbuild_{programs,contrib,tests,examples}=true
+        - |
+          travis_retry curl -o ~/ninja.zip -L 'https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-linux.zip' &&
+          unzip ~/ninja.zip -d ~/.local/bin
+        - |
+          travis_retry curl -o ~/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' &&
+          python3 ~/get-pip.py --user &&
+          pip3 install --user meson
+      script:
+        - |
+          meson setup \
+            --buildtype=debug \
+            -Db_lundef=false \
+            -Dauto_features=enabled \
+            -Ddefault_library=both \
+            -Dbin_programs=true \
+            -Dbin_contrib=true \
+            -Dbin_tests=true \
+            -Dbin_examples=true \
             contrib/meson build
         - pushd build
         - DESTDIR=./staging ninja install
-- 
cgit v0.12


From f3ec519f594ca78ba22001f416db84ea2da41fa1 Mon Sep 17 00:00:00 2001
From: Max Dymond 
Date: Sun, 30 Jun 2019 20:16:03 +0100
Subject: Remove unnecessary call to Makefile.inc

---
 ossfuzz/Makefile | 2 --
 1 file changed, 2 deletions(-)

diff --git a/ossfuzz/Makefile b/ossfuzz/Makefile
index 7812c41..bd01123 100644
--- a/ossfuzz/Makefile
+++ b/ossfuzz/Makefile
@@ -35,8 +35,6 @@ LZ4_CFLAGS  = $(CFLAGS) $(DEBUGFLAGS) $(MOREFLAGS)
 LZ4_CXXFLAGS = $(CXXFLAGS) $(DEBUGFLAGS) $(MOREFLAGS)
 LZ4_CPPFLAGS = $(CPPFLAGS) -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_
 
-include ../Makefile.inc
-
 # Include a rule to build the static library if calling this target
 # directly.
 $(LZ4DIR)/liblz4.a:
-- 
cgit v0.12


From 4e87942529323e8db18707689651fa279b13ac82 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sun, 30 Jun 2019 13:59:49 -0700
Subject: frametest: added LZ4F decoder noise test

---
 tests/frametest.c | 265 +++++++++++++++++++++++++++++++++++++-----------------
 tests/fuzzer.c    |  12 +--
 2 files changed, 188 insertions(+), 89 deletions(-)

diff --git a/tests/frametest.c b/tests/frametest.c
index bf95beb..32a023c 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -41,7 +41,7 @@
 #include      /* strcmp */
 #include        /* clock_t, clock(), CLOCKS_PER_SEC */
 #include 
-#include "lz4frame.h"   /* include multiple times to test correctness/safety */
+#include "lz4frame.h"   /* included multiple times to test correctness/safety */
 #include "lz4frame.h"
 #define LZ4F_STATIC_LINKING_ONLY
 #include "lz4frame.h"
@@ -150,8 +150,7 @@ static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, dou
             size_t const length = MIN(lengthRand, bufferSize - pos);
             size_t const end = pos + length;
             while (pos < end) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5);
-        }
-    }
+    }   }
 }
 
 
@@ -703,8 +702,8 @@ int basicTests(U32 seed, double compressibility)
         while (ip < iend) {
             unsigned nbBits = FUZ_rand(&randState) % maxBits;
             size_t iSize = (FUZ_rand(&randState) & ((1< (size_t)(iend-ip)) iSize = iend-ip;
+            size_t oSize = (size_t)(oend-op);
+            if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip);
             CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
             op += oSize;
             ip += iSize;
@@ -722,8 +721,8 @@ int basicTests(U32 seed, double compressibility)
         while (ip < iend) {
             unsigned const nbBits = FUZ_rand(&randState) % maxBits;
             size_t iSize = (FUZ_rand(&randState) & ((1< (size_t)(iend-ip)) iSize = iend-ip;
+            size_t oSize = (size_t)(oend-op);
+            if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip);
             CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
             op += oSize;
             ip += iSize;
@@ -739,7 +738,7 @@ int basicTests(U32 seed, double compressibility)
         while (ip < iend) {
             size_t iSize = 10;
             size_t oSize = 10;
-            if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
+            if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip);
             CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
             op += oSize;
             ip += iSize;
@@ -763,15 +762,17 @@ _output_error:
 }
 
 
-static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, unsigned nonContiguous)
+typedef enum { o_contiguous, o_noncontiguous, o_overwrite } o_scenario_e;
+
+static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, o_scenario_e o_scenario)
 {
     size_t p=0;
     const BYTE* b1=(const BYTE*)buff1;
     const BYTE* b2=(const BYTE*)buff2;
     DISPLAY("locateBuffDiff: looking for error position \n");
-    if (nonContiguous) {
-        DISPLAY("mode %u: non-contiguous output (%u bytes), cannot search \n",
-                nonContiguous, (unsigned)size);
+    if (o_scenario != o_contiguous) {
+        DISPLAY("mode %i: non-contiguous output (%u bytes), cannot search \n",
+                (int)o_scenario, (unsigned)size);
         return;
     }
     while (p < size && b1[p]==b2[p]) p++;
@@ -780,38 +781,143 @@ static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, un
     }
 }
 
+#   define EXIT_MSG(...) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
+                           DISPLAY(" (seed %u, test nb %u)  \n", seed, testNb); exit(1); }
+#   undef CHECK
+#   define CHECK(cond, ...) { if (cond) { EXIT_MSG(__VA_ARGS__); } }
+
+
+size_t test_lz4f_decompression_wBuffers(
+          const void* cSrc, size_t cSize,
+                void* dst, size_t dstCapacity, o_scenario_e o_scenario,
+          const void* srcRef, size_t decompressedSize,
+                U64 crcOrig,
+                U32* const randState,
+                LZ4F_dctx* const dCtx,
+                U32 seed, U32 testNb)
+{
+    const BYTE* ip = (const BYTE*)cSrc;
+    const BYTE* const iend = ip + cSize;
+
+    BYTE* op = (BYTE*)dst;
+    BYTE* const oend = op + dstCapacity;
+
+    unsigned const suggestedBits = FUZ_highbit((U32)cSize);
+    unsigned const maxBits = MAX(3, suggestedBits);
+    size_t totalOut = 0;
+    size_t moreToFlush = 0;
+    XXH64_state_t xxh64;
+    XXH64_reset(&xxh64, 1);
+    assert(ip < iend);
+    while (ip < iend) {
+        unsigned const nbBitsI = (FUZ_rand(randState) % (maxBits-1)) + 1;
+        unsigned const nbBitsO = (FUZ_rand(randState) % (maxBits)) + 1;
+        size_t const iSizeCand = (FUZ_rand(randState) & ((1< 2x4MB to test large blocks */
-    void* srcBuffer = NULL;
-    size_t const compressedBufferSize = LZ4F_compressFrameBound(srcDataLength, NULL) + 4 MB;  /* needs some margin */
+    size_t const CNBufferLength = 9 MB;  /* needs to be > 2x4MB to test large blocks */
+    void* CNBuffer = NULL;
+    size_t const compressedBufferSize = LZ4F_compressFrameBound(CNBufferLength, NULL) + 4 MB;  /* needs some margin */
     void* compressedBuffer = NULL;
     void* decodedBuffer = NULL;
     U32 coreRand = seed;
     LZ4F_decompressionContext_t dCtx = NULL;
+    LZ4F_decompressionContext_t dCtxNoise = NULL;
     LZ4F_compressionContext_t cCtx = NULL;
     clock_t const startClock = clock();
     clock_t const clockDuration = duration_s * CLOCKS_PER_SEC;
-#   undef CHECK
-#   define EXIT_MSG(...) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
-                           DISPLAY(" (seed %u, test nb %u)  \n", seed, testNb); goto _output_error; }
-#   define CHECK(cond, ...) { if (cond) { EXIT_MSG(__VA_ARGS__); } }
 
     /* Create buffers */
     {   size_t const creationStatus = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
         CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); }
+    {   size_t const creationStatus = LZ4F_createDecompressionContext(&dCtxNoise, LZ4F_VERSION);
+        CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); }
     {   size_t const creationStatus = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION);
         CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); }
-    srcBuffer = malloc(srcDataLength);
-    CHECK(srcBuffer==NULL, "srcBuffer Allocation failed");
+    CNBuffer = malloc(CNBufferLength);
+    CHECK(CNBuffer==NULL, "CNBuffer Allocation failed");
     compressedBuffer = malloc(compressedBufferSize);
     CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed");
-    decodedBuffer = calloc(1, srcDataLength);   /* calloc avoids decodedBuffer being considered "garbage" by scan-build */
+    decodedBuffer = calloc(1, CNBufferLength);   /* calloc avoids decodedBuffer being considered "garbage" by scan-build */
     CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed");
-    FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand);
+    FUZ_fillCompressibleNoiseBuffer(CNBuffer, CNBufferLength, compressibility, &coreRand);
 
     /* jump to requested testNb */
     for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand);   /* sync randomizer */
@@ -819,10 +925,10 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
     /* main fuzzer test loop */
     for ( ; (testNb < nbTests) || (clockDuration > FUZ_GetClockSpan(startClock)) ; testNb++) {
         U32 randState = coreRand ^ prime1;
-        unsigned const srcBits = (FUZ_rand(&randState) % (FUZ_highbit((U32)(srcDataLength-1)) - 1)) + 1;
+        unsigned const srcBits = (FUZ_rand(&randState) % (FUZ_highbit((U32)(CNBufferLength-1)) - 1)) + 1;
         size_t const srcSize = (FUZ_rand(&randState) & ((1< %s)",
+                                        (int)result, LZ4F_getErrorName(result));
         }
-    }
+
+#if 1
+        /* insert noise into src */
+        {   U32 const maxNbBits = FUZ_highbit((U32)cSize);
+            size_t pos = 0;
+            for (;;) {
+                /* keep some original src */
+                {   U32 const nbBits = FUZ_rand(&randState) % maxNbBits;
+                    size_t const mask = (1<= cSize) break;
+                /* add noise */
+                {   U32 const nbBitsCodes = FUZ_rand(&randState) % maxNbBits;
+                    U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0;
+                    size_t const mask = (1<='0') && (*argument<='9')) {
                         nbTests *= 10;
-                        nbTests += *argument - '0';
+                        nbTests += (unsigned)(*argument - '0');
                         argument++;
                     }
                     break;
@@ -1071,7 +1170,7 @@ int main(int argc, const char** argv)
                             case '6':
                             case '7':
                             case '8':
-                            case '9': duration *= 10; duration += *argument++ - '0'; continue;
+                            case '9': duration *= 10; duration += (U32)(*argument++ - '0'); continue;
                         }
                         break;
                     }
@@ -1083,7 +1182,7 @@ int main(int argc, const char** argv)
                     seedset=1;
                     while ((*argument>='0') && (*argument<='9')) {
                         seed *= 10;
-                        seed += *argument - '0';
+                        seed += (U32)(*argument - '0');
                         argument++;
                     }
                     break;
@@ -1092,7 +1191,7 @@ int main(int argc, const char** argv)
                     testNb=0;
                     while ((*argument>='0') && (*argument<='9')) {
                         testNb *= 10;
-                        testNb += *argument - '0';
+                        testNb += (unsigned)(*argument - '0');
                         argument++;
                     }
                     break;
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 7b6e929..74bc5a0 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1528,8 +1528,8 @@ int main(int argc, const char** argv)
     U32 seed = 0;
     int seedset = 0;
     int argNb;
-    int nbTests = NB_ATTEMPTS;
-    int testNb = 0;
+    unsigned nbTests = NB_ATTEMPTS;
+    unsigned testNb = 0;
     int proba = FUZ_COMPRESSIBILITY_DEFAULT;
     int use_pause = 0;
     const char* programName = argv[0];
@@ -1567,7 +1567,7 @@ int main(int argc, const char** argv)
                     nbTests = 0; duration = 0;
                     while ((*argument>='0') && (*argument<='9')) {
                         nbTests *= 10;
-                        nbTests += *argument - '0';
+                        nbTests += (unsigned)(*argument - '0');
                         argument++;
                     }
                     break;
@@ -1590,7 +1590,7 @@ int main(int argc, const char** argv)
                             case '6':
                             case '7':
                             case '8':
-                            case '9': duration *= 10; duration += *argument++ - '0'; continue;
+                            case '9': duration *= 10; duration += (U32)(*argument++ - '0'); continue;
                         }
                         break;
                     }
@@ -1601,7 +1601,7 @@ int main(int argc, const char** argv)
                     seed=0; seedset=1;
                     while ((*argument>='0') && (*argument<='9')) {
                         seed *= 10;
-                        seed += *argument - '0';
+                        seed += (U32)(*argument - '0');
                         argument++;
                     }
                     break;
@@ -1611,7 +1611,7 @@ int main(int argc, const char** argv)
                     testNb=0;
                     while ((*argument>='0') && (*argument<='9')) {
                         testNb *= 10;
-                        testNb += *argument - '0';
+                        testNb += (unsigned)(*argument - '0');
                         argument++;
                     }
                     break;
-- 
cgit v0.12


From 89e96e55ffab100f9893f2b6c12b4b1e26def17e Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sun, 30 Jun 2019 14:57:12 -0700
Subject: updated frametest

so that noisy src decompression
doesn't generate output
nor fails when decompression fails (which is expected).
---
 tests/frametest.c | 43 +++++++++++++++++++++++++++----------------
 1 file changed, 27 insertions(+), 16 deletions(-)

diff --git a/tests/frametest.c b/tests/frametest.c
index 32a023c..6d690f2 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -766,18 +766,20 @@ typedef enum { o_contiguous, o_noncontiguous, o_overwrite } o_scenario_e;
 
 static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, o_scenario_e o_scenario)
 {
-    size_t p=0;
-    const BYTE* b1=(const BYTE*)buff1;
-    const BYTE* b2=(const BYTE*)buff2;
-    DISPLAY("locateBuffDiff: looking for error position \n");
-    if (o_scenario != o_contiguous) {
-        DISPLAY("mode %i: non-contiguous output (%u bytes), cannot search \n",
-                (int)o_scenario, (unsigned)size);
-        return;
-    }
-    while (p < size && b1[p]==b2[p]) p++;
-    if (p != size) {
-        DISPLAY("Error at pos %i/%i : %02X != %02X \n", (int)p, (int)size, b1[p], b2[p]);
+    if (displayLevel >= 5) {
+        size_t p=0;
+        const BYTE* b1=(const BYTE*)buff1;
+        const BYTE* b2=(const BYTE*)buff2;
+        DISPLAY("locateBuffDiff: looking for error position \n");
+        if (o_scenario != o_contiguous) {
+            DISPLAY("mode %i: non-contiguous output (%u bytes), cannot search \n",
+                    (int)o_scenario, (unsigned)size);
+            return;
+        }
+        while (p < size && b1[p]==b2[p]) p++;
+        if (p != size) {
+            DISPLAY("Error at pos %i/%i : %02X != %02X \n", (int)p, (int)size, b1[p], b2[p]);
+        }
     }
 }
 
@@ -825,6 +827,8 @@ size_t test_lz4f_decompression_wBuffers(
         if (o_scenario == o_overwrite) dOptions.stableDst = 0;  /* overwrite mode */
         if (op + oSizeMax < oend) op[oSizeMax] = mark;
 
+        DISPLAYLEVEL(7, "dstCapacity=%u,  presentedInput=%u \n", (unsigned)oSize, (unsigned)iSize);
+
         /* read data from byte-exact buffer to catch out-of-bound reads */
         {   void* const iBuffer = malloc(iSizeMax);
             assert(iBuffer != NULL);
@@ -832,6 +836,7 @@ size_t test_lz4f_decompression_wBuffers(
             moreToFlush = LZ4F_decompress(dCtx, op, &oSize, iBuffer, &iSize, &dOptions);
             free(iBuffer);
         }
+        DISPLAYLEVEL(7, "oSize=%u,  readSize=%u \n", (unsigned)oSize, (unsigned)iSize);
 
         if (op + oSizeMax < oend) {
             CHECK(op[oSizeMax] != mark, "op[oSizeMax] = %02X != %02X : "
@@ -846,8 +851,14 @@ size_t test_lz4f_decompression_wBuffers(
         totalOut += oSize;
         op += oSize;
         ip += iSize;
-        op += (o_scenario == o_noncontiguous);  /* create a gap between consecutive output */
+        if (o_scenario == o_noncontiguous) {
+            if (op == oend) return LZ4F_ERROR_GENERIC;  /* can theoretically happen with bogus data */
+            op++; /* create a gap between consecutive output */
+        }
         if (o_scenario==o_overwrite) op = (BYTE*)dst;   /* overwrite destination */
+        if ( (op == oend) /* no more room for output; can happen with bogus input */
+          && (iSize == 0)) /* no input consumed */
+            break;
     }
     if (moreToFlush != 0) return LZ4F_ERROR_decompressionFailed;
     if (totalOut) {  /* otherwise, it's a skippable frame */
@@ -870,7 +881,7 @@ size_t test_lz4f_decompression(const void* cSrc, size_t cSize,
     o_scenario_e const o_scenario = (o_scenario_e)(FUZ_rand(randState) % 3);   /* 0 : contiguous; 1 : non-contiguous; 2 : dst overwritten */
     /* tighten dst buffer conditions */
     size_t const dstCapacity = (o_scenario == o_noncontiguous) ?
-                               (decompressedSize * 2) + 64 :
+                               (decompressedSize * 2) + 128 :
                                decompressedSize;
     size_t result;
     void* const dstBuffer = malloc(dstCapacity);
@@ -1023,7 +1034,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
 
 
         /* multi-segments decompression */
-        printf("normal decompression \n");
+        DISPLAYLEVEL(6, "normal decompression \n");
         {   size_t result = test_lz4f_decompression(compressedBuffer, cSize, srcStart, srcSize, crcOrig, &randState, dCtx, seed, testNb);
             CHECK (LZ4F_isError(result), "multi-segment decompression failed (error %i => %s)",
                                         (int)result, LZ4F_getErrorName(result));
@@ -1053,7 +1064,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
         }   }   }
 
         /* test decompression on noisy src */
-        printf("noisy decompression \n");
+        DISPLAYLEVEL(6, "noisy decompression \n");
         test_lz4f_decompression(compressedBuffer, cSize, srcStart, srcSize, crcOrig, &randState, dCtxNoise, seed, testNb);
         /* note : we don't analyze result here : it probably failed, which is expected.
          * We just check for potential out-of-bound reads and writes. */
-- 
cgit v0.12


From 2cacdd21423010406dbafaad710dfa9c567f77c1 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sun, 30 Jun 2019 15:36:32 -0700
Subject: fix minor cppcheck warnings

---
 tests/frametest.c | 2 +-
 tests/fuzzer.c    | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/frametest.c b/tests/frametest.c
index 6d690f2..69dd5aa 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -1236,7 +1236,7 @@ int main(int argc, const char** argv)
     DISPLAY("Seed = %u\n", seed);
     if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
 
-    if (nbTests<=0) nbTests=1;
+    nbTests += (nbTests==0);  /* avoid zero */
 
     if (testNb==0) result = basicTests(seed, ((double)proba) / 100);
     if (result) return 1;
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 74bc5a0..8a095c4 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -861,7 +861,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_DISPLAYTEST("LZ4_decompress_safe_usingDict with a too small output buffer");
         {   U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
             if ((U32)blockSize > missingBytes) {
-                decodedBuffer[blockSize-missingBytes] = 0;
+                decodedBuffer[(U32)blockSize-missingBytes] = 0;
                 ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-missingBytes, dict, dictSize);
                 FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%u byte)", missingBytes);
                 FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize);
@@ -1646,7 +1646,7 @@ int main(int argc, const char** argv)
 
     if ((seedset==0) && (testNb==0)) { FUZ_unitTests(LZ4HC_CLEVEL_DEFAULT); FUZ_unitTests(LZ4HC_CLEVEL_OPT_MIN); }
 
-    if (nbTests<=0) nbTests=1;
+    nbTests += (nbTests==0);  /* avoid zero */
 
     {   int const result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100, duration);
         if (use_pause) {
-- 
cgit v0.12


From bb5c34a875a892077e60582ac27898489bfd9d14 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 1 Jul 2019 09:01:43 -0700
Subject: bumped version number to v1.9.2

to reduce risks that future bug reports in `dev` branch report `v1.9.1` as the failing version.
---
 doc/lz4_manual.html      |  4 ++--
 doc/lz4frame_manual.html |  4 ++--
 lib/lz4.h                |  2 +-
 programs/lz4.1           |  2 +-
 programs/lz4io.c         | 16 ++++++++--------
 5 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index 18daa05..a477584 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -1,10 +1,10 @@
 
 
 
-1.9.1 Manual
+1.9.2 Manual
 
 
-
1.9.1 Manual

+
1.9.2 Manual

 

 
Contents

 
    
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index 2cf4f03..72f27c8 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -1,10 +1,10 @@
 
 
 
-1.9.1 Manual
+1.9.2 Manual
 
 
-
    1.9.1 Manual
    
+
    1.9.2 Manual
    
 
    
 
    Contents
    
 
      
diff --git a/lib/lz4.h b/lib/lz4.h
index ad93d00..32108e2 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -100,7 +100,7 @@ extern "C" {
 /*------   Version   ------*/
 #define LZ4_VERSION_MAJOR    1    /* for breaking interface changes  */
 #define LZ4_VERSION_MINOR    9    /* for new (non-breaking) interface capabilities */
-#define LZ4_VERSION_RELEASE  1    /* for tweaks, bug-fixes, or development */
+#define LZ4_VERSION_RELEASE  2    /* for tweaks, bug-fixes, or development */
 
 #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
 
diff --git a/programs/lz4.1 b/programs/lz4.1
index ad0c12c..3ca017a 100644
--- a/programs/lz4.1
+++ b/programs/lz4.1
@@ -1,5 +1,5 @@
 .
-.TH "LZ4" "1" "April 2019" "lz4 1.9.1" "User Commands"
+.TH "LZ4" "1" "July 2019" "lz4 1.9.2" "User Commands"
 .
 .SH "NAME"
 \fBlz4\fR \- lz4, unlz4, lz4cat \- Compress or decompress \.lz4 files
diff --git a/programs/lz4io.c b/programs/lz4io.c
index d03aa6d..d818535 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -219,8 +219,8 @@ size_t LZ4IO_setBlockSizeID(LZ4IO_prefs_t* const prefs, unsigned bsid)
     static const unsigned minBlockSizeID = 4;
     static const unsigned maxBlockSizeID = 7;
     if ((bsid < minBlockSizeID) || (bsid > maxBlockSizeID)) return 0;
-    prefs->blockSizeId = bsid;
-    prefs->blockSize = blockSizeTable[prefs->blockSizeId-minBlockSizeID];
+    prefs->blockSizeId = (int)bsid;
+    prefs->blockSize = blockSizeTable[(unsigned)prefs->blockSizeId-minBlockSizeID];
     return prefs->blockSize;
 }
 
@@ -237,7 +237,7 @@ size_t LZ4IO_setBlockSize(LZ4IO_prefs_t* const prefs, size_t blockSize)
     while (blockSize >>= 2)
         bsid++;
     if (bsid < 7) bsid = 7;
-    prefs->blockSizeId = bsid-3;
+    prefs->blockSizeId = (int)(bsid-3);
     return prefs->blockSize;
 }
 
@@ -417,7 +417,7 @@ int LZ4IO_compressFilename_Legacy(LZ4IO_prefs_t* const prefs, const char* input_
 
     /* Allocate Memory */
     in_buff = (char*)malloc(LEGACY_BLOCKSIZE);
-    out_buff = (char*)malloc(outBuffSize + 4);
+    out_buff = (char*)malloc((size_t)outBuffSize + 4);
     if (!in_buff || !out_buff)
         EXM_THROW(21, "Allocation error : not enough memory");
 
@@ -898,7 +898,7 @@ static unsigned long long LZ4IO_decodeLegacyStream(LZ4IO_prefs_t* const prefs, F
     unsigned storedSkips = 0;
 
     /* Allocate Memory */
-    char* const in_buff  = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE));
+    char* const in_buff  = (char*)malloc((size_t)LZ4_compressBound(LEGACY_BLOCKSIZE));
     char* const out_buff = (char*)malloc(LEGACY_BLOCKSIZE);
     if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory");
 
@@ -922,11 +922,11 @@ static unsigned long long LZ4IO_decodeLegacyStream(LZ4IO_prefs_t* const prefs, F
           if (sizeCheck!=blockSize) EXM_THROW(52, "Read error : cannot access compressed block !"); }
 
         /* Decode Block */
-        {   int const decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE);
+        {   int const decodeSize = LZ4_decompress_safe(in_buff, out_buff, (int)blockSize, LEGACY_BLOCKSIZE);
             if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !");
-            streamSize += decodeSize;
+            streamSize += (unsigned long long)decodeSize;
             /* Write Block */
-            storedSkips = LZ4IO_fwriteSparse(prefs, foutput, out_buff, decodeSize, storedSkips); /* success or die */
+            storedSkips = LZ4IO_fwriteSparse(prefs, foutput, out_buff, (size_t)decodeSize, storedSkips); /* success or die */
     }   }
     if (ferror(finput)) EXM_THROW(54, "Read error : ferror");
 
-- 
cgit v0.12


From 9ea2835eaa2da7415e3c97a010585e12705d6220 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 1 Jul 2019 10:55:54 -0700
Subject: CircleCI : reduced test duration

Ideally, we want to make CircleCI a "fast" test environment, with short feedback loop (~5mn).
We are still far from this goal.

This patch starts this process by removing "long" tasks which are non-essential and redundant with travisCI.

It also acknowledges that parallelism is broken.
The script should be more heavily updated to support parallelism,
which might be important to support its goal.
---
 .circleci/config.yml | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 3abcbc1..7f03d1a 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -10,7 +10,10 @@ version: 2
 jobs:
   build:
     working_directory: ~/lz4/lz4
-    parallelism: 2
+    # Parallelism is broken in this file : it just plays the same tests twice.
+    # The script will have to be modified to support parallelism properly
+    # In the meantime, set it to 1.
+    parallelism: 1
     shell: /bin/bash --login
     # CircleCI 2.0 does not support environment variables that refer to each other the same way as 1.0 did.
     # If any of these refer to each other, rewrite them so that they don't or see https://circleci.com/docs/2.0/env-vars/#interpolating-environment-variables-to-set-other-environment-variables .
@@ -42,25 +45,20 @@ jobs:
     # This is based on your 1.0 configuration file or project settings
     - run: CFLAGS= make clangtest && make clean
     - run: g++ -v; make gpptest     && make clean
-    - run: gcc -v; make c_standards && make clean
     - run: gcc -v; g++ -v; make ctocpptest && make clean
     - run: gcc-5 -v; CC=gcc-5 CFLAGS="-O2 -Werror" make check && make clean
     - run: gcc-5 -v; CC=gcc-5 CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean
-    - run: gcc-6 -v; CC=gcc-6 make c_standards && make clean
     - run: gcc-6 -v; CC=gcc-6 MOREFLAGS="-O2 -Werror" make check  && make clean
     - run: make cmake               && make clean
     - run: make -C tests test-lz4
     - run: make -C tests test-lz4c
     - run: make -C tests test-frametest
-    - run: make -C tests test-fullbench
     - run: make -C tests test-fuzzer && make clean
     - run: make -C lib all          && make clean
     - run: pyenv global 3.4.4; make versionsTest MOREFLAGS=-I/usr/include/x86_64-linux-gnu && make clean
     - run: make travis-install      && make clean
     - run: gcc -v; CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean
-    - run: make usan                && make clean
     - run: clang -v; make staticAnalyze && make clean
-    - run: make -C tests test-mem && make clean
     - run: make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static                  && make clean
     - run: make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS=-m64 && make clean
     - run: make platformTest CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static                  && make clean
-- 
cgit v0.12


From 0e6ff83d091e151ad510845bb3b4ebb42314f6a3 Mon Sep 17 00:00:00 2001
From: Amine Choukir 
Date: Wed, 3 Jul 2019 11:50:38 +0200
Subject: Update blockStreaming_doubleBuffer.md

---
 examples/blockStreaming_doubleBuffer.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/examples/blockStreaming_doubleBuffer.md b/examples/blockStreaming_doubleBuffer.md
index 3027ea3..38dc2e8 100644
--- a/examples/blockStreaming_doubleBuffer.md
+++ b/examples/blockStreaming_doubleBuffer.md
@@ -1,7 +1,7 @@
 # LZ4 Streaming API Example : Double Buffer
 by *Takayuki Matsuoka*
 
-`blockStreaming_doubleBuffer.c` is LZ4 Straming API example which implements double buffer (de)compression.
+`blockStreaming_doubleBuffer.c` is LZ4 Streaming API example which implements double buffer (de)compression.
 
 Please note :
 
@@ -76,10 +76,10 @@ so it just compress the line without dependencies and generates compressed block
 After that, write {Out#1} to the file.
 
 Next, read second block to double buffer's second page. And compress it.
-In this time, LZ4 can use dependency to Block#1 to improve compression ratio.
+This time, LZ4 can use dependency to Block#1 to improve compression ratio.
 This dependency is called "Prefix mode".
 
-Next, read third block to double buffer's *first* page. And compress it.
+Next, read third block to double buffer's *first* page, and compress it.
 Also this time, LZ4 can use dependency to Block#2.
 This dependency is called "External Dictonaly mode".
 
-- 
cgit v0.12


From 12e5841e762c69e9daa79d6f6d6a5f4302492479 Mon Sep 17 00:00:00 2001
From: Sylvestre Ledru 
Date: Thu, 4 Jul 2019 18:13:36 +0200
Subject: Remove an useless declaration

---
 lib/lz4.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index d121e29..38ad6ab 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1149,7 +1149,7 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
             return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
-        if (inputSize < LZ4_64Klimit) {;
+        if (inputSize < LZ4_64Klimit) {
             return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
         } else {
             const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32;
-- 
cgit v0.12


From 658ab8fca1c7bbc191d63c17ee751344a591f3ce Mon Sep 17 00:00:00 2001
From: Hamid Zare 
Date: Thu, 11 Jul 2019 14:34:52 -0700
Subject: changed the input text to something more compression friendly

---
 examples/simple_buffer.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/simple_buffer.c b/examples/simple_buffer.c
index 027fc85..62409d0 100644
--- a/examples/simple_buffer.c
+++ b/examples/simple_buffer.c
@@ -38,7 +38,7 @@ int main(void) {
 
   /* Compression */
   // We'll store some text into a variable pointed to by *src to be compressed later.
-  const char* const src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit.";
+  const char* const src = "Lorem ipsum dolor sit amet, hhhhhhhhhhhhhhhhhhhhhhh";
   // The compression function needs to know how many bytes exist.  Since we're using a string, we can use strlen() + 1 (for \0).
   const int src_size = (int)(strlen(src) + 1);
   // LZ4 provides a function that will tell you the maximum size of compressed output based on input data via LZ4_compressBound().
-- 
cgit v0.12


From 771a7192d6aa46e5c136ded366a216af60df3583 Mon Sep 17 00:00:00 2001
From: Hamid Zare 
Date: Thu, 11 Jul 2019 14:39:29 -0700
Subject: print the compression ratio

---
 examples/simple_buffer.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/examples/simple_buffer.c b/examples/simple_buffer.c
index 62409d0..ea57022 100644
--- a/examples/simple_buffer.c
+++ b/examples/simple_buffer.c
@@ -55,7 +55,8 @@ int main(void) {
   if (compressed_data_size <= 0)
     run_screaming("A 0 or negative result from LZ4_compress_default() indicates a failure trying to compress the data. ", 1);
   if (compressed_data_size > 0)
-    printf("We successfully compressed some data!\n");
+    printf("We successfully compressed some data! Ratio: %.2f\n",
+        (float) compressed_data_size/src_size);
   // Not only does a positive return_value mean success, the value returned == the number of bytes required.
   // You can use this to realloc() *compress_data to free up memory, if desired.  We'll do so just to demonstrate the concept.
   compressed_data = (char *)realloc(compressed_data, (size_t)compressed_data_size);
-- 
cgit v0.12


From f1e8e806e0dd753eb48e40272e620f747c7b723e Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 11 Jul 2019 17:29:16 -0700
Subject: keep the "lorem ipsum" topic of the example string

but make it compressible
---
 examples/simple_buffer.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/simple_buffer.c b/examples/simple_buffer.c
index ea57022..6afc62a 100644
--- a/examples/simple_buffer.c
+++ b/examples/simple_buffer.c
@@ -38,7 +38,7 @@ int main(void) {
 
   /* Compression */
   // We'll store some text into a variable pointed to by *src to be compressed later.
-  const char* const src = "Lorem ipsum dolor sit amet, hhhhhhhhhhhhhhhhhhhhhhh";
+  const char* const src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Lorem ipsum dolor site amat.";
   // The compression function needs to know how many bytes exist.  Since we're using a string, we can use strlen() + 1 (for \0).
   const int src_size = (int)(strlen(src) + 1);
   // LZ4 provides a function that will tell you the maximum size of compressed output based on input data via LZ4_compressBound().
-- 
cgit v0.12


From 8ac954aa71edd5c13a0b9f6ab57eaece75efc28d Mon Sep 17 00:00:00 2001
From: Hitatm 
Date: Mon, 15 Jul 2019 22:53:46 +0800
Subject: bugfix: correctly control the offset < LZ4_DISTANCE_MAX,when change
 the value of LZ4_DISTANCE_MAX,

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 936f739..46c20bc 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -228,7 +228,7 @@ LZ4HC_InsertAndGetWiderMatch (
     const U32 dictLimit = hc4->dictLimit;
     const BYTE* const lowPrefixPtr = base + dictLimit;
     const U32 ipIndex = (U32)(ip - base);
-    const U32 lowestMatchIndex = (hc4->lowLimit + 64 KB > ipIndex) ? hc4->lowLimit : ipIndex - LZ4_DISTANCE_MAX;
+    const U32 lowestMatchIndex = (hc4->lowLimit + (LZ4_DISTANCE_MAX + 1) > ipIndex) ? hc4->lowLimit : ipIndex - LZ4_DISTANCE_MAX;
     const BYTE* const dictBase = hc4->dictBase;
     int const lookBackLength = (int)(ip-iLowLimit);
     int nbAttempts = maxNbAttempts;
-- 
cgit v0.12


From 6654c2cd3bd49dfc6b6bb3447bcc01799ea35ac3 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 15 Jul 2019 12:11:34 -0700
Subject: ensure conformance with custom LZ4_DISTANCE_MAX

It's now possible to select a custom LZ4_DISTANCE_MAX at compile time,
provided it's <= 65535.

However, in some cases (when compressing in byU16 mode),
the new distance wasn't respected,
as it used to implied that it was necessarily within range.

Added a distance check for this case.
Also : added a new TravisCI test which ensures that
custom LZ4_DISTANCE_MAX compiles correctly
and compresses correctly (relying on `assert()` to find outsized offsets).
---
 .travis.yml       |  4 ++++
 lib/README.md     |  4 ++--
 lib/lz4.c         | 13 +++++++++----
 tests/Makefile    |  2 +-
 tests/frametest.c |  3 ++-
 5 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index af5cf0f..bd29630 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -33,6 +33,10 @@ matrix:
       script:
         - CC=clang MOREFLAGS=-fsanitize=address make -C tests test-frametest test-fuzzer
 
+    - name: Custom LZ4_DISTANCE_MAX
+      script:
+        - MOREFLAGS=-DLZ4_DISTANCE_MAX=8000 make check
+
     - name: (Precise) g++ and clang CMake test
       dist: precise
       script:
diff --git a/lib/README.md b/lib/README.md
index cf1505f..cba2c34 100644
--- a/lib/README.md
+++ b/lib/README.md
@@ -56,8 +56,8 @@ The following build macro can be selected at compilation time :
 - `LZ4_DISTANCE_MAX` : control the maximum offset that the compressor will allow.
   Set to 65535 by default, which is the maximum value supported by lz4 format.
   Reducing maximum distance will reduce opportunities for LZ4 to find matches,
-  hence will produce worse the compression ratio.
-  However, a smaller max distance may allow compatibility with specific decoders using limited memory budget.
+  hence will produce a worse compression ratio.
+  However, a smaller max distance can allow compatibility with specific decoders using limited memory budget.
   This build macro only influences the compressed output of the compressor.
 
 - `LZ4_DISABLE_DEPRECATE_WARNINGS` : invoking a deprecated function will make the compiler generate a warning.
diff --git a/lib/lz4.c b/lib/lz4.c
index 38ad6ab..abbdd97 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -413,7 +413,8 @@ static const int LZ4_minLength = (MFLIMIT+1);
 #define MB *(1 <<20)
 #define GB *(1U<<30)
 
-#if (LZ4_DISTANCE_MAX > 65535)   /* max supported by LZ4 format */
+#define LZ4_DISTANCE_ABSOLUTE_MAX 65535
+#if (LZ4_DISTANCE_MAX > LZ4_DISTANCE_ABSOLUTE_MAX)   /* max supported by LZ4 format */
 #  error "LZ4_DISTANCE_MAX is too big : must be <= 65535"
 #endif
 
@@ -915,10 +916,14 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 forwardH = LZ4_hashPosition(forwardIp, tableType);
                 LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
 
+                DEBUGLOG(7, "candidate at pos=%u  (offset=%u \n", matchIndex, current - matchIndex);
                 if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) { continue; }    /* match outside of valid area */
                 assert(matchIndex < current);
-                if ((tableType != byU16) && (matchIndex+LZ4_DISTANCE_MAX < current)) { continue; } /* too far */
-                if (tableType == byU16) { assert((current - matchIndex) <= LZ4_DISTANCE_MAX); } /* too_far presumed impossible with byU16 */
+                if ( ((tableType != byU16) || (LZ4_DISTANCE_MAX < LZ4_DISTANCE_ABSOLUTE_MAX))
+                  && (matchIndex+LZ4_DISTANCE_MAX < current)) {
+                    continue;
+                } /* too far */
+                assert((current - matchIndex) <= LZ4_DISTANCE_MAX);  /* match now expected within distance */
 
                 if (LZ4_read32(match) == LZ4_read32(ip)) {
                     if (maybe_extMem) offset = current - matchIndex;
@@ -1082,7 +1087,7 @@ _next_match:
             LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
             assert(matchIndex < current);
             if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1)
-              && ((tableType==byU16) ? 1 : (matchIndex+LZ4_DISTANCE_MAX >= current))
+              && (((tableType==byU16) && (LZ4_DISTANCE_MAX == LZ4_DISTANCE_ABSOLUTE_MAX)) ? 1 : (matchIndex+LZ4_DISTANCE_MAX >= current))
               && (LZ4_read32(match) == LZ4_read32(ip)) ) {
                 token=op++;
                 *token=0;
diff --git a/tests/Makefile b/tests/Makefile
index 65713ef..422baba 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -448,7 +448,7 @@ test-fuzzer32: CFLAGS += -m32
 test-fuzzer32: test-fuzzer
 
 test-frametest: frametest
-	./frametest $(FUZZER_TIME)
+	./frametest -v $(FUZZER_TIME)
 
 test-frametest32: CFLAGS += -m32
 test-frametest32: test-frametest
diff --git a/tests/frametest.c b/tests/frametest.c
index 69dd5aa..91813bb 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -46,6 +46,7 @@
 #define LZ4F_STATIC_LINKING_ONLY
 #include "lz4frame.h"
 #include "lz4frame.h"
+#define LZ4_STATIC_LINKING_ONLY  /* LZ4_DISTANCE_MAX */
 #include "lz4.h"        /* LZ4_VERSION_STRING */
 #define XXH_STATIC_LINKING_ONLY
 #include "xxhash.h"     /* XXH64 */
@@ -540,7 +541,7 @@ int basicTests(U32 seed, double compressibility)
                                               cdict, NULL) );
         DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
                         (unsigned)dictSize, (unsigned)cSizeWithDict);
-        if (cSizeWithDict >= cSizeNoDict) goto _output_error;  /* must be more efficient */
+        if ((LZ4_DISTANCE_MAX > dictSize) && (cSizeWithDict >= cSizeNoDict)) goto _output_error;  /* must be more efficient */
         crcOrig = XXH64(CNBuffer, dictSize, 0);
 
         DISPLAYLEVEL(3, "LZ4F_decompress_usingDict : ");
-- 
cgit v0.12


From 725cb0aafdf78b550c52618fe5cea1fadd278881 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 12 Jul 2019 18:36:28 -0700
Subject: [lz4] Fix bugs in partial decoding

* Partial decoding could read a few bytes beyond the end of the input
* Partial decoding returned an error with an empty output buffer
---
 lib/lz4.c | 49 +++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 41 insertions(+), 8 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 38ad6ab..7a4c6aa 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1642,7 +1642,11 @@ LZ4_decompress_generic(
 
         /* Special cases */
         assert(lowPrefix <= op);
-        if ((endOnInput) && (unlikely(outputSize==0))) { return ((srcSize==1) && (*ip==0)) ? 0 : -1; } /* Empty output buffer */
+        if ((endOnInput) && (unlikely(outputSize==0))) {
+            /* Empty output buffer */
+            if (partialDecoding) return 0;
+            return ((srcSize==1) && (*ip==0)) ? 0 : -1;
+        }
         if ((!endOnInput) && (unlikely(outputSize==0))) { return (*ip==0 ? 1 : -1); }
         if ((endOnInput) && unlikely(srcSize==0)) { return -1; }
 
@@ -1850,21 +1854,50 @@ LZ4_decompress_generic(
             if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) )
               || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) )
             {
+                /* We've either hit the input parsing restriction or the output parsing restriction.
+                 * If we've hit the input parsing condition then this must be the last sequence.
+                 * If we've hit the output parsing condition then we are either using partialDecoding
+                 * or we've hit the output parsing condition.
+                 */
                 if (partialDecoding) {
-                    if (cpy > oend) { cpy = oend; assert(op<=oend); length = (size_t)(oend-op); }  /* Partial decoding : stop in the middle of literal segment */
-                    if ((endOnInput) && (ip+length > iend)) { goto _output_error; } /* Error : read attempt beyond end of input buffer */
+                    /* Since we are partial decoding we may be in this block because of the output parsing
+                     * restriction, which is not valid since the output buffer is allowed to be undersized.
+                     */
+                    assert(endOnInput);
+                    /* If we're in this block because of the input parsing condition, then we must be on the
+                     * last sequence (or invalid), so we must check that we exactly consume the input.
+                     */
+                    if ((ip+length>iend-(2+1+LASTLITERALS)) && (ip+length != iend)) { goto _output_error; }
+                    assert(ip+length <= iend);
+                    /* We are finishing in the middle of a literals segment.
+                     * Break after the copy.
+                     */
+                    if (cpy > oend) {
+                        cpy = oend;
+                        assert(op<=oend);
+                        length = (size_t)(oend-op);
+                    }
+                    assert(ip+length <= iend);
                 } else {
-                    if ((!endOnInput) && (cpy != oend)) { goto _output_error; }   /* Error : block decoding must stop exactly there */
-                    if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) { goto _output_error; } /* Error : input must be consumed */
+                    /* We must be on the last sequence because of the parsing limitations so check
+                     * that we exactly regenerate the original size (must be exact when !endOnInput).
+                     */
+                    if ((!endOnInput) && (cpy != oend)) { goto _output_error; }
+                     /* We must be on the last sequence (or invalid) because of the parsing limitations
+                      * so check that we exactly consume the input and don't overrun the output buffer.
+                      */
+                    if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) { goto _output_error; }
                 }
                 memmove(op, ip, length);  /* supports overlapping memory regions, which only matters for in-place decompression scenarios */
                 ip += length;
                 op += length;
-                if (!partialDecoding || (cpy == oend)) {
-                    /* Necessarily EOF, due to parsing restrictions */
+                /* Necessarily EOF when !partialDecoding. When partialDecoding
+                 * it is EOF if we've either filled the output buffer or hit
+                 * the input parsing restriction.
+                 */
+                if (!partialDecoding || (cpy == oend) || (ip == iend)) {
                     break;
                 }
-
             } else {
                 LZ4_wildCopy8(op, ip, cpy);   /* may overwrite up to WILDCOPYLENGTH beyond cpy */
                 ip += length; op = cpy;
-- 
cgit v0.12


From 3c40db8d258716b9efcfb46fa6dc29de6e43e616 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 12 Jul 2019 19:27:00 -0700
Subject: [ossfuzz] Improve the fuzzers

* Run more decompression variants
* Round trip the compression fuzzer and do partial decompression as well
* Add a compression fuzzer that compresses into a smaller output buffer
  and test the destSize variant

These fuzzers caught 2 bugs that were fixed in the previous commit.
* Input buffer over-read in partial decompress
* Partial decompress fails if output size is 0
---
 ossfuzz/Makefile            |  1 +
 ossfuzz/compress_fuzzer.c   | 52 +++++++++++++++++++-------
 ossfuzz/decompress_fuzzer.c | 60 ++++++++++++++++++++++--------
 ossfuzz/fuzz.h              | 48 ++++++++++++++++++++++++
 ossfuzz/fuzz_helpers.h      | 89 +++++++++++++++++++++++++++++++++++++++++++++
 ossfuzz/round_trip_fuzzer.c | 50 +++++++++++++++++++++++++
 ossfuzz/standaloneengine.c  |  2 +-
 ossfuzz/testinput.h         | 15 --------
 8 files changed, 273 insertions(+), 44 deletions(-)
 create mode 100644 ossfuzz/fuzz.h
 create mode 100644 ossfuzz/fuzz_helpers.h
 create mode 100644 ossfuzz/round_trip_fuzzer.c
 delete mode 100644 ossfuzz/testinput.h

diff --git a/ossfuzz/Makefile b/ossfuzz/Makefile
index bd01123..4d24944 100644
--- a/ossfuzz/Makefile
+++ b/ossfuzz/Makefile
@@ -57,3 +57,4 @@ $(LZ4DIR)/liblz4.a:
 
 .PHONY: clean
 clean: compress_fuzzer_clean decompress_fuzzer_clean
+	$(MAKE) -C $(LZ4DIR) clean
diff --git a/ossfuzz/compress_fuzzer.c b/ossfuzz/compress_fuzzer.c
index 3908534..7021624 100644
--- a/ossfuzz/compress_fuzzer.c
+++ b/ossfuzz/compress_fuzzer.c
@@ -1,25 +1,51 @@
+/**
+ * This fuzz target attempts to compress the fuzzed data with the simple
+ * compression function with an output buffer that may be too small to
+ * ensure that the compressor never crashes.
+ */
+
 #include 
 #include 
 #include 
-#include "lz4.h"
+#include 
 
-#define CHECK(COND)   if (!(COND)) { abort(); }
+#include "fuzz_helpers.h"
+#include "lz4.h"
 
 int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 {
-  size_t const compressed_dest_size = LZ4_compressBound(size);
-  char *const dest_buffer = (char *)malloc(compressed_dest_size);
+    uint32_t seed = FUZZ_seed(&data, &size);
+    size_t const dstCapacity = FUZZ_rand32(&seed, 0, LZ4_compressBound(size));
+    char* const dst = (char*)malloc(dstCapacity);
+    char* const rt = (char*)malloc(size);
+
+    FUZZ_ASSERT(dst);
+    FUZZ_ASSERT(rt);
 
-  CHECK(dest_buffer != NULL);
+    /* If compression succeeds it must round trip correctly. */
+    {
+        int const dstSize = LZ4_compress_default((const char*)data, dst,
+                                                 size, dstCapacity);
+        if (dstSize > 0) {
+            int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size);
+            FUZZ_ASSERT_MSG(rtSize == size, "Incorrect regenerated size");
+            FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!");
+        }
+    }
 
-  // Allocation succeeded, try compressing the incoming data.
-  int result = LZ4_compress_default((const char*)data,
-                                    dest_buffer,
-                                    size,
-                                    compressed_dest_size);
-  CHECK(result != 0);
+    if (dstCapacity > 0) {
+        /* Compression succeeds and must round trip correctly. */
+        int compressedSize = size;
+        int const dstSize = LZ4_compress_destSize((const char*)data, dst,
+                                                  &compressedSize, dstCapacity);
+        FUZZ_ASSERT(dstSize > 0);
+        int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size);
+        FUZZ_ASSERT_MSG(rtSize == compressedSize, "Incorrect regenerated size");
+        FUZZ_ASSERT_MSG(!memcmp(data, rt, compressedSize), "Corruption!");
+    }
 
-  free(dest_buffer);
+    free(dst);
+    free(rt);
 
-  return 0;
+    return 0;
 }
diff --git a/ossfuzz/decompress_fuzzer.c b/ossfuzz/decompress_fuzzer.c
index e6e14c4..0267c93 100644
--- a/ossfuzz/decompress_fuzzer.c
+++ b/ossfuzz/decompress_fuzzer.c
@@ -1,28 +1,58 @@
+/**
+ * This fuzz target attempts to decompress the fuzzed data with the simple
+ * decompression function to ensure the decompressor never crashes.
+ */
+
 #include 
 #include 
 #include 
-#include "lz4.h"
+#include 
 
-#define CHECK(COND)   if (!(COND)) { abort(); }
+#include "fuzz_helpers.h"
+#include "lz4.h"
 
 int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 {
-  // TODO: Size input buffer pseudo-randomly based on seed extracted from input
-  size_t const buffer_size = 10 * 1024 * 1024;
-  char *const dest_buffer = (char *)malloc(buffer_size);
 
-  CHECK(dest_buffer != NULL);
+    uint32_t seed = FUZZ_seed(&data, &size);
+    size_t const dstCapacity = FUZZ_rand32(&seed, 0, 4 * size);
+    size_t const smallDictSize = size + 1;
+    size_t const largeDictSize = 64 * 1024 - 1;
+    size_t const dictSize = MAX(smallDictSize, largeDictSize);
+    char* const dst = (char*)malloc(dstCapacity);
+    char* const dict = (char*)malloc(dictSize + size);
+    char* const largeDict = dict;
+    char* const dataAfterDict = dict + dictSize;
+    char* const smallDict = dataAfterDict - smallDictSize;
 
-  // Allocation succeeded, try decompressing the incoming data.
-  int result = LZ4_decompress_safe((const char*)data,
-                                   dest_buffer,
-                                   size,
-                                   buffer_size);
+    FUZZ_ASSERT(dst);
+    FUZZ_ASSERT(dict);
 
-  // Ignore the result of decompression.
-  (void)result;
+    /* Prepare the dictionary. The data doesn't matter for decompression. */
+    memset(dict, 0, dictSize);
+    memcpy(dataAfterDict, data, size);
 
-  free(dest_buffer);
+    /* Decompress using each possible dictionary configuration. */
+    /* No dictionary. */
+    LZ4_decompress_safe_usingDict((char const*)data, dst, size,
+                                  dstCapacity, NULL, 0);
+    /* Small external dictonary. */
+    LZ4_decompress_safe_usingDict((char const*)data, dst, size,
+                                  dstCapacity, smallDict, smallDictSize);
+    /* Large external dictionary. */
+    LZ4_decompress_safe_usingDict((char const*)data, dst, size,
+                                  dstCapacity, largeDict, largeDictSize);
+    /* Small prefix. */
+    LZ4_decompress_safe_usingDict((char const*)dataAfterDict, dst, size,
+                                  dstCapacity, smallDict, smallDictSize);
+    /* Large prefix. */
+    LZ4_decompress_safe_usingDict((char const*)data, dst, size,
+                                  dstCapacity, largeDict, largeDictSize);
+    /* Partial decompression. */
+    LZ4_decompress_safe_partial((char const*)data, dst, size,
+                                dstCapacity, dstCapacity);
+    free(dst);
+    free(dict);
 
-  return 0;
+    return 0;
 }
diff --git a/ossfuzz/fuzz.h b/ossfuzz/fuzz.h
new file mode 100644
index 0000000..eefac63
--- /dev/null
+++ b/ossfuzz/fuzz.h
@@ -0,0 +1,48 @@
+/**
+ * Fuzz target interface.
+ * Fuzz targets have some common parameters passed as macros during compilation.
+ * Check the documentation for each individual fuzzer for more parameters.
+ *
+ * @param FUZZ_RNG_SEED_SIZE:
+ *        The number of bytes of the source to look at when constructing a seed
+ *        for the deterministic RNG. These bytes are discarded before passing
+ *        the data to lz4 functions. Every fuzzer initializes the RNG exactly
+ *        once before doing anything else, even if it is unused.
+ *        Default: 4.
+ * @param LZ4_DEBUG:
+ *        This is a parameter for the lz4 library. Defining `LZ4_DEBUG=1`
+ *        enables assert() statements in the lz4 library. Higher levels enable
+ *        logging, so aren't recommended. Defining `LZ4_DEBUG=1` is
+ *        recommended.
+ * @param LZ4_FORCE_MEMORY_ACCESS:
+ *        This flag controls how the zstd library accesses unaligned memory.
+ *        It can be undefined, or 0 through 2. If it is undefined, it selects
+ *        the method to use based on the compiler. If testing with UBSAN set
+ *        MEM_FORCE_MEMORY_ACCESS=0 to use the standard compliant method.
+ * @param FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+ *        This is the canonical flag to enable deterministic builds for fuzzing.
+ *        Changes to zstd for fuzzing are gated behind this define.
+ *        It is recommended to define this when building zstd for fuzzing.
+ */
+
+#ifndef FUZZ_H
+#define FUZZ_H
+
+#ifndef FUZZ_RNG_SEED_SIZE
+#  define FUZZ_RNG_SEED_SIZE 4
+#endif
+
+#include 
+#include 
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ossfuzz/fuzz_helpers.h b/ossfuzz/fuzz_helpers.h
new file mode 100644
index 0000000..626f209
--- /dev/null
+++ b/ossfuzz/fuzz_helpers.h
@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+
+/**
+ * Helper functions for fuzzing.
+ */
+
+#ifndef FUZZ_HELPERS_H
+#define FUZZ_HELPERS_H
+
+#include "fuzz.h"
+#include "xxhash.h"
+#include 
+#include 
+#include 
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+
+#define FUZZ_QUOTE_IMPL(str) #str
+#define FUZZ_QUOTE(str) FUZZ_QUOTE_IMPL(str)
+
+/**
+ * Asserts for fuzzing that are always enabled.
+ */
+#define FUZZ_ASSERT_MSG(cond, msg)                                             \
+  ((cond) ? (void)0                                                            \
+          : (fprintf(stderr, "%s: %u: Assertion: `%s' failed. %s\n", __FILE__, \
+                     __LINE__, FUZZ_QUOTE(cond), (msg)),                       \
+             abort()))
+#define FUZZ_ASSERT(cond) FUZZ_ASSERT_MSG((cond), "");
+
+#if defined(__GNUC__)
+#define FUZZ_STATIC static __inline __attribute__((unused))
+#elif defined(__cplusplus) ||                                                  \
+    (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+#define FUZZ_STATIC static inline
+#elif defined(_MSC_VER)
+#define FUZZ_STATIC static __inline
+#else
+#define FUZZ_STATIC static
+#endif
+
+/**
+ * Deterministically constructs a seed based on the fuzz input.
+ * Consumes up to the first FUZZ_RNG_SEED_SIZE bytes of the input.
+ */
+FUZZ_STATIC uint32_t FUZZ_seed(uint8_t const **src, size_t* size) {
+    uint8_t const *data = *src;
+    size_t const toHash = MIN(FUZZ_RNG_SEED_SIZE, *size);
+    *size -= toHash;
+    *src += toHash;
+    return XXH32(data, toHash, 0);
+}
+
+#define FUZZ_rotl32(x, r) (((x) << (r)) | ((x) >> (32 - (r))))
+
+FUZZ_STATIC uint32_t FUZZ_rand(uint32_t *state) {
+    static const uint32_t prime1 = 2654435761U;
+    static const uint32_t prime2 = 2246822519U;
+    uint32_t rand32 = *state;
+    rand32 *= prime1;
+    rand32 += prime2;
+    rand32 = FUZZ_rotl32(rand32, 13);
+    *state = rand32;
+    return rand32 >> 5;
+}
+
+/* Returns a random numer in the range [min, max]. */
+FUZZ_STATIC uint32_t FUZZ_rand32(uint32_t *state, uint32_t min, uint32_t max) {
+    uint32_t random = FUZZ_rand(state);
+    return min + (random % (max - min + 1));
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ossfuzz/round_trip_fuzzer.c b/ossfuzz/round_trip_fuzzer.c
new file mode 100644
index 0000000..3a66e80
--- /dev/null
+++ b/ossfuzz/round_trip_fuzzer.c
@@ -0,0 +1,50 @@
+/**
+ * This fuzz target performs a lz4 round-trip test (compress & decompress),
+ * compares the result with the original, and calls abort() on corruption.
+ */
+
+#include 
+#include 
+#include 
+#include 
+
+#include "fuzz_helpers.h"
+#include "lz4.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
+{
+    uint32_t seed = FUZZ_seed(&data, &size);
+    size_t const dstCapacity = LZ4_compressBound(size);
+    char* const dst = (char*)malloc(dstCapacity);
+    char* const rt = (char*)malloc(size);
+
+    FUZZ_ASSERT(dst);
+    FUZZ_ASSERT(rt);
+
+    /* Compression must succeed and round trip correctly. */
+    int const dstSize = LZ4_compress_default((const char*)data, dst,
+                                             size, dstCapacity);
+    FUZZ_ASSERT(dstSize > 0);
+
+    int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size);
+    FUZZ_ASSERT_MSG(rtSize == size, "Incorrect size");
+    FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!");
+
+    /* Partial decompression must succeed. */
+    {
+        size_t const partialCapacity = FUZZ_rand32(&seed, 0, size);
+        char* const partial = (char*)malloc(partialCapacity);
+        FUZZ_ASSERT(partial);
+        int const partialSize = LZ4_decompress_safe_partial(
+                dst, partial, dstSize, partialCapacity, partialCapacity);
+        FUZZ_ASSERT(partialSize >= 0);
+        FUZZ_ASSERT_MSG(partialSize == partialCapacity, "Incorrect size");
+        FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!");
+        free(partial);
+    }
+
+    free(dst);
+    free(rt);
+
+    return 0;
+}
diff --git a/ossfuzz/standaloneengine.c b/ossfuzz/standaloneengine.c
index 175360e..6afeffd 100644
--- a/ossfuzz/standaloneengine.c
+++ b/ossfuzz/standaloneengine.c
@@ -2,7 +2,7 @@
 #include 
 #include 
 
-#include "testinput.h"
+#include "fuzz.h"
 
 /**
  * Main procedure for standalone fuzzing engine.
diff --git a/ossfuzz/testinput.h b/ossfuzz/testinput.h
deleted file mode 100644
index 0e50a3c..0000000
--- a/ossfuzz/testinput.h
+++ /dev/null
@@ -1,15 +0,0 @@
-#ifndef TESTINPUT_H_INCLUDED
-#define TESTINPUT_H_INCLUDED
-
-#include 
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size);
-
-#if defined(__cplusplus)
-}
-#endif
-#endif
-- 
cgit v0.12


From 690009e2c2f9e5dcb0d40e7c0c40610ce6006eda Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 17 Jul 2019 11:07:24 -0700
Subject: [LZ4_compress_destSize] Allow 2 more bytes of match length

---
 lib/lz4.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 5b03e3d..1e80c98 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1016,7 +1016,7 @@ _next_match:
                 (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) {
                 if (outputDirective == fillOutput) {
                     /* Match description too long : reduce it */
-                    U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 2 - 1 - LASTLITERALS) * 255;
+                    U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 1 - LASTLITERALS) * 255;
                     ip -= matchCode - newMatchCode;
                     matchCode = newMatchCode;
                 } else {
-- 
cgit v0.12


From 6bc6f836a18d1f8fd05c8fc2b42f1d800bc25de1 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 17 Jul 2019 11:28:38 -0700
Subject: [LZ4_compress_destSize] Fix rare data corruption bug

---
 lib/lz4.c | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/lib/lz4.c b/lib/lz4.c
index 1e80c98..461644d 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -648,6 +648,18 @@ LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tab
     return LZ4_hash4(LZ4_read32(p), tableType);
 }
 
+static void LZ4_clearHash(U32 h, void* tableBase, tableType_t const tableType)
+{
+    switch (tableType)
+    {
+    default: /* fallthrough */
+    case clearedTable: { /* illegal! */ assert(0); return; }
+    case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = NULL; return; }
+    case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = 0; return; }
+    case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = 0; return; }
+    }
+}
+
 static void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t const tableType)
 {
     switch (tableType)
@@ -848,6 +860,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     for ( ; ; ) {
         const BYTE* match;
         BYTE* token;
+        const BYTE* filledIp;
 
         /* Find a match */
         if (tableType == byPtr) {
@@ -934,6 +947,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         }
 
         /* Catch up */
+        filledIp = ip;
         while (((ip>anchor) & (match > lowLimit)) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; }
 
         /* Encode Literals */
@@ -1018,7 +1032,21 @@ _next_match:
                     /* Match description too long : reduce it */
                     U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 1 - LASTLITERALS) * 255;
                     ip -= matchCode - newMatchCode;
+                    assert(newMatchCode < matchCode);
                     matchCode = newMatchCode;
+                    if (unlikely(ip < filledIp)) {
+                        /* We have already filled up to filledIp so if ip ends up less than filledIp
+                         * we have positions in the hash table beyond the current position. This is
+                         * a problem if we reuse the hash table. So we have to remove these positions
+                         * from the hash table.
+                         */
+                        const BYTE* ptr;
+                        DEBUGLOG(5, "Clearing %u positions", (U32)(filledIp - ip));
+                        for (ptr = ip + 1; ptr <= filledIp; ++ptr) {
+                            U32 const h = LZ4_hashPosition(ptr, tableType);
+                            LZ4_clearHash(h, cctx->hashTable, tableType);
+                        }
+                    }
                 } else {
                     assert(outputDirective == limitedOutput);
                     return 0;   /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */
@@ -1038,6 +1066,8 @@ _next_match:
             } else
                 *token += (BYTE)(matchCode);
         }
+        /* Ensure we have enough space for the last literals. */
+        assert(!(outputDirective == fillOutput && op + 1 + LASTLITERALS > olimit));
 
         anchor = ip;
 
-- 
cgit v0.12


From 13a2d9e34ffc4170720ce417c73e396d0ac1471a Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 17 Jul 2019 11:50:47 -0700
Subject: [LZ4_compress_destSize] Fix overflow condition

---
 lib/lz4.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 461644d..74a9247 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1027,7 +1027,7 @@ _next_match:
             }
 
             if ((outputDirective) &&    /* Check output buffer overflow */
-                (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) {
+                (unlikely(op + (1 + LASTLITERALS) + (matchCode+240)/255 > olimit)) ) {
                 if (outputDirective == fillOutput) {
                     /* Match description too long : reduce it */
                     U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 1 - LASTLITERALS) * 255;
-- 
cgit v0.12


From 99d925f997923f24492d2c68d932bc958f98ace9 Mon Sep 17 00:00:00 2001
From: dooxe 
Date: Thu, 18 Jul 2019 11:25:43 +0200
Subject: Added BUNDLE DESTINATION in CMakeLists.txt so that it works with
 newer versions of cmake

---
 contrib/cmake_unofficial/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/contrib/cmake_unofficial/CMakeLists.txt b/contrib/cmake_unofficial/CMakeLists.txt
index b09c4fb..42d92ea 100644
--- a/contrib/cmake_unofficial/CMakeLists.txt
+++ b/contrib/cmake_unofficial/CMakeLists.txt
@@ -172,6 +172,7 @@ if(NOT LZ4_BUNDLED_MODE)
   include(GNUInstallDirs)
 
   install(TARGETS ${LZ4_PROGRAMS_BUILT}
+    BUNDLE	DESTINATION "${CMAKE_INSTALL_BINDIR}"
     RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}")
   install(TARGETS ${LZ4_LIBRARIES_BUILT}
     LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
-- 
cgit v0.12


From 369fb3900cbc73543f1bab276ca1b82abe402937 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Thu, 18 Jul 2019 12:41:12 -0400
Subject: Fix Data Corruption Bug when Streaming with an Attached Dict in HC
 Mode

This diff fixes an issue in which we failed to clear the `dictCtx` in HC
compression. The `dictCtx` is not supposed to be used when an `extDict` is
present: matches found in the `dictCtx` do not account for the presence of an
`extDict` segment, and their offsets are therefore miscalculated when one is
present. This can lead to data corruption.

This diff clears the `dictCtx` whenever setting an `extDict`.

This issue was uncovered by @terrelln's fuzzing work.
---
 lib/lz4hc.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 46c20bc..d9e55a0 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -998,6 +998,11 @@ static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBl
     if (ctxPtr->end >= ctxPtr->base + ctxPtr->dictLimit + 4)
         LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   /* Referencing remaining dictionary content */
 
+    /* cannot reference an extDict and a dictCtx at the same time */
+    if (ctxPtr->dictCtx != NULL) {
+        ctxPtr->dictCtx = NULL;
+    }
+
     /* Only one memory segment for extDict, so any previous extDict is lost at this stage */
     ctxPtr->lowLimit  = ctxPtr->dictLimit;
     ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base);
-- 
cgit v0.12


From 40943ba0c90483dbe229299c39d048ae79f08246 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Thu, 18 Jul 2019 13:35:12 -0400
Subject: Unconditionally Clear `dictCtx`

---
 lib/lz4hc.c | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index d9e55a0..29288a5 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -998,11 +998,6 @@ static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBl
     if (ctxPtr->end >= ctxPtr->base + ctxPtr->dictLimit + 4)
         LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   /* Referencing remaining dictionary content */
 
-    /* cannot reference an extDict and a dictCtx at the same time */
-    if (ctxPtr->dictCtx != NULL) {
-        ctxPtr->dictCtx = NULL;
-    }
-
     /* Only one memory segment for extDict, so any previous extDict is lost at this stage */
     ctxPtr->lowLimit  = ctxPtr->dictLimit;
     ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base);
@@ -1010,6 +1005,9 @@ static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBl
     ctxPtr->base = newBlock - ctxPtr->dictLimit;
     ctxPtr->end  = newBlock;
     ctxPtr->nextToUpdate = ctxPtr->dictLimit;   /* match referencing will resume from there */
+
+    /* cannot reference an extDict and a dictCtx at the same time */
+    ctxPtr->dictCtx = NULL;
 }
 
 static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr,
-- 
cgit v0.12


From 7c32101c655d93b61fc212dcd512b87119dd7333 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Thu, 18 Jul 2019 12:20:29 -0700
Subject: [LZ4_compress_destSize] Fix off-by-one error in fix

The next match is looking at the current ip, not the next ip,
so it needs to be cleared as well.

Credit to OSS-Fuzz
---
 lib/lz4.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 74a9247..70424b9 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1042,7 +1042,7 @@ _next_match:
                          */
                         const BYTE* ptr;
                         DEBUGLOG(5, "Clearing %u positions", (U32)(filledIp - ip));
-                        for (ptr = ip + 1; ptr <= filledIp; ++ptr) {
+                        for (ptr = ip; ptr <= filledIp; ++ptr) {
                             U32 const h = LZ4_hashPosition(ptr, tableType);
                             LZ4_clearHash(h, cctx->hashTable, tableType);
                         }
-- 
cgit v0.12


From 9b258abd93e19dece681ea59db9c31e1e27635ff Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 17 Jul 2019 17:33:31 -0700
Subject: [fuzz] Add a streaming round trip fuzzer

---
 ossfuzz/round_trip_stream_fuzzer.c | 217 +++++++++++++++++++++++++++++++++++++
 1 file changed, 217 insertions(+)
 create mode 100644 ossfuzz/round_trip_stream_fuzzer.c

diff --git a/ossfuzz/round_trip_stream_fuzzer.c b/ossfuzz/round_trip_stream_fuzzer.c
new file mode 100644
index 0000000..4facfc9
--- /dev/null
+++ b/ossfuzz/round_trip_stream_fuzzer.c
@@ -0,0 +1,217 @@
+/**
+ * This fuzz target performs a lz4 streaming round-trip test
+ * (compress & decompress), compares the result with the original, and calls
+ * abort() on corruption.
+ */
+
+#include 
+#include 
+#include 
+#include 
+
+#include "fuzz_helpers.h"
+#define LZ4_STATIC_LINKING_ONLY
+#include "lz4.h"
+
+typedef struct {
+  char const* buf;
+  size_t size;
+  size_t pos;
+} const_cursor_t;
+
+typedef struct {
+  char* buf;
+  size_t size;
+  size_t pos;
+} cursor_t;
+
+typedef struct {
+  LZ4_stream_t* cstream;
+  LZ4_streamDecode_t* dstream;
+  const_cursor_t data;
+  cursor_t compressed;
+  cursor_t roundTrip;
+  uint32_t seed;
+} state_t;
+
+cursor_t cursor_create(size_t size)
+{
+  cursor_t cursor;
+  cursor.buf = (char*)malloc(size);
+  cursor.size = size;
+  cursor.pos = 0;
+  FUZZ_ASSERT(cursor.buf);
+  return cursor;
+}
+
+void cursor_free(cursor_t cursor)
+{
+    free(cursor.buf);
+}
+
+state_t state_create(char const* data, size_t size, uint32_t seed)
+{
+    state_t state;
+
+    state.seed = seed;
+
+    state.data.buf = (char const*)data;
+    state.data.size = size;
+    state.data.pos = 0;
+
+    /* Extra margin because we are streaming. */
+    state.compressed = cursor_create(1024 + 2 * LZ4_compressBound(size));
+    state.roundTrip = cursor_create(size);
+
+    state.cstream = LZ4_createStream();
+    FUZZ_ASSERT(state.cstream);
+    state.dstream = LZ4_createStreamDecode();
+    FUZZ_ASSERT(state.dstream);
+
+    return state;
+}
+
+void state_free(state_t state)
+{
+    cursor_free(state.compressed);
+    cursor_free(state.roundTrip);
+    LZ4_freeStream(state.cstream);
+    LZ4_freeStreamDecode(state.dstream);
+}
+
+static void state_reset(state_t* state, uint32_t seed)
+{
+    LZ4_resetStream_fast(state->cstream);
+    LZ4_setStreamDecode(state->dstream, NULL, 0);
+    state->data.pos = 0;
+    state->compressed.pos = 0;
+    state->roundTrip.pos = 0;
+    state->seed = seed;
+}
+
+static void state_decompress(state_t* state, char const* src, int srcSize)
+{
+    char* dst = state->roundTrip.buf + state->roundTrip.pos;
+    int const dstCapacity = state->roundTrip.size - state->roundTrip.pos;
+    int const dSize = LZ4_decompress_safe_continue(state->dstream, src, dst,
+                                                   srcSize, dstCapacity);
+    FUZZ_ASSERT(dSize >= 0);
+    state->roundTrip.pos += dSize;
+}
+
+static void state_checkRoundTrip(state_t const* state)
+{
+    char const* data = state->data.buf;
+    size_t const size = state->data.size;
+    FUZZ_ASSERT_MSG(size == state->roundTrip.pos, "Incorrect size!");
+    FUZZ_ASSERT_MSG(!memcmp(data, state->roundTrip.buf, size), "Corruption!");
+}
+
+/**
+ * Picks a dictionary size and trims the dictionary off of the data.
+ */
+static size_t state_trimDict(state_t* state)
+{
+    /* 64 KB is the max dict size, allow slightly beyond that to test trim. */
+    uint32_t maxDictSize = MIN(70 * 1024, state->data.size);
+    size_t const dictSize = FUZZ_rand32(&state->seed, 0, maxDictSize);
+    FUZZ_ASSERT(state->compressed.pos == 0);
+    state->compressed.pos += dictSize;
+    return dictSize;
+}
+
+static void state_prefixRoundTrip(state_t* state)
+{
+    while (state->data.pos != state->data.size) {
+        char const* src = state->data.buf + state->data.pos;
+        char* dst = state->compressed.buf + state->compressed.pos;
+        int const srcRemaining = state->data.size - state->data.pos;
+        int const srcSize = FUZZ_rand32(&state->seed, 0, srcRemaining);
+        int const dstCapacity = state->compressed.size - state->compressed.pos;
+        int const cSize = LZ4_compress_fast_continue(state->cstream, src, dst,
+                                                     srcSize, dstCapacity, 0);
+        FUZZ_ASSERT(cSize > 0);
+        state->data.pos += srcSize;
+        state->compressed.pos += cSize;
+        state_decompress(state, dst, cSize);
+    }
+}
+
+static void state_extDictRoundTrip(state_t* state)
+{
+    int i = 0;
+    cursor_t data2 = cursor_create(state->data.size);
+    memcpy(data2.buf, state->data.buf, state->data.size);
+    while (state->data.pos != state->data.size) {
+        char const* data = (i++ & 1) ? state->data.buf : data2.buf;
+        char const* src = data + state->data.pos;
+        char* dst = state->compressed.buf + state->compressed.pos;
+        int const srcRemaining = state->data.size - state->data.pos;
+        int const srcSize = FUZZ_rand32(&state->seed, 0, srcRemaining);
+        int const dstCapacity = state->compressed.size - state->compressed.pos;
+        int const cSize = LZ4_compress_fast_continue(state->cstream, src, dst,
+                                                     srcSize, dstCapacity, 0);
+        FUZZ_ASSERT(cSize > 0);
+        state->data.pos += srcSize;
+        state->compressed.pos += cSize;
+        state_decompress(state, dst, cSize);
+    }
+    cursor_free(data2);
+}
+
+static void state_randomRoundTrip(state_t* state)
+{
+    if (FUZZ_rand32(&state->seed, 0, 1)) {
+      state_prefixRoundTrip(state);
+    } else {
+      state_extDictRoundTrip(state);
+    }
+}
+
+static void state_loadDictRoundTrip(state_t* state)
+{
+    char const* dict = state->compressed.buf;
+    size_t const dictSize = state_trimDict(state);
+    LZ4_loadDict(state->cstream, dict, dictSize);
+    LZ4_setStreamDecode(state->dstream, dict, dictSize);
+    state_randomRoundTrip(state);
+}
+
+static void state_attachDictRoundTrip(state_t* state)
+{
+    char const* dict = state->compressed.buf;
+    size_t const dictSize = state_trimDict(state);
+    LZ4_stream_t* dictStream = LZ4_createStream();
+    LZ4_loadDict(dictStream, dict, dictSize);
+    LZ4_attach_dictionary(state->cstream, dictStream);
+    LZ4_setStreamDecode(state->dstream, dict, dictSize);
+    state_randomRoundTrip(state);
+    LZ4_freeStream(dictStream);
+}
+
+typedef void (*round_trip_t)(state_t* state);
+
+round_trip_t roundTrips[] = {
+  &state_prefixRoundTrip,
+  &state_extDictRoundTrip,
+  &state_loadDictRoundTrip,
+  &state_attachDictRoundTrip,
+};
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
+{
+    uint32_t seed = FUZZ_seed(&data, &size);
+    state_t state = state_create((char const*)data, size, seed);
+    const int n = sizeof(roundTrips) / sizeof(round_trip_t);
+    int i;
+
+    for (i = 0; i < n; ++i) {
+        roundTrips[i](&state);
+        state_checkRoundTrip(&state);
+        state_reset(&state, seed);
+    }
+
+    state_free(state);
+
+    return 0;
+}
-- 
cgit v0.12


From 399a80d48e4352aeae99eb5cbd799bcee6978a3d Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 17 Jul 2019 17:41:41 -0700
Subject: [fuzzer] Update scripts for new fuzzers

---
 ossfuzz/Makefile     | 8 ++++++++
 ossfuzz/ossfuzz.sh   | 4 ++--
 ossfuzz/travisoss.sh | 5 +----
 3 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/ossfuzz/Makefile b/ossfuzz/Makefile
index 4d24944..2bb40ec 100644
--- a/ossfuzz/Makefile
+++ b/ossfuzz/Makefile
@@ -35,6 +35,14 @@ LZ4_CFLAGS  = $(CFLAGS) $(DEBUGFLAGS) $(MOREFLAGS)
 LZ4_CXXFLAGS = $(CXXFLAGS) $(DEBUGFLAGS) $(MOREFLAGS)
 LZ4_CPPFLAGS = $(CPPFLAGS) -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_
 
+FUZZERS := \
+	compress_fuzzer \
+	decompress_fuzzer \
+	round_trip_fuzzer \
+	round_trip_stream_fuzzer
+
+all: $(FUZZERS)
+
 # Include a rule to build the static library if calling this target
 # directly.
 $(LZ4DIR)/liblz4.a:
diff --git a/ossfuzz/ossfuzz.sh b/ossfuzz/ossfuzz.sh
index a76b0d6..9782286 100755
--- a/ossfuzz/ossfuzz.sh
+++ b/ossfuzz/ossfuzz.sh
@@ -16,8 +16,8 @@ echo "OUT: $OUT"
 export MAKEFLAGS+="-j$(nproc)"
 
 pushd ossfuzz
-make V=1 compress_fuzzer decompress_fuzzer
+make V=1 all
 popd
 
 # Copy the fuzzers to the target directory.
-cp -v ossfuzz/compress_fuzzer ossfuzz/decompress_fuzzer $OUT/
+cp -v ossfuzz/*_fuzzer $OUT/
diff --git a/ossfuzz/travisoss.sh b/ossfuzz/travisoss.sh
index 3b2f26f..5ea884c 100755
--- a/ossfuzz/travisoss.sh
+++ b/ossfuzz/travisoss.sh
@@ -8,10 +8,7 @@ git clone https://github.com/google/oss-fuzz.git /tmp/ossfuzz
 if [[ ! -d /tmp/ossfuzz/projects/lz4 ]]
 then
     echo "Could not find the lz4 project in ossfuzz"
-
-    # Exit with a success code while the lz4 project is not expected to exist
-    # on oss-fuzz.
-    exit 0
+    exit 1
 fi
 
 # Modify the oss-fuzz Dockerfile so that we're checking out the current branch on travis.
-- 
cgit v0.12


From 675ef9a9fc9899667d0fe9b7b2a66e402e870a6d Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 17 Jul 2019 18:31:44 -0700
Subject: [fuzz] Add HC fuzzers for round trip, compress, and streaming

---
 ossfuzz/Makefile                   |   4 +-
 ossfuzz/compress_hc_fuzzer.c       |  57 ++++++++++++++++++++
 ossfuzz/fuzz_helpers.h             |   9 +++-
 ossfuzz/round_trip_hc_fuzzer.c     |  39 ++++++++++++++
 ossfuzz/round_trip_stream_fuzzer.c | 107 +++++++++++++++++++++++++++++++++----
 5 files changed, 202 insertions(+), 14 deletions(-)
 create mode 100644 ossfuzz/compress_hc_fuzzer.c
 create mode 100644 ossfuzz/round_trip_hc_fuzzer.c

diff --git a/ossfuzz/Makefile b/ossfuzz/Makefile
index 2bb40ec..9974b81 100644
--- a/ossfuzz/Makefile
+++ b/ossfuzz/Makefile
@@ -39,7 +39,9 @@ FUZZERS := \
 	compress_fuzzer \
 	decompress_fuzzer \
 	round_trip_fuzzer \
-	round_trip_stream_fuzzer
+	round_trip_stream_fuzzer \
+	compress_hc_fuzzer \
+	round_trip_hc_fuzzer
 
 all: $(FUZZERS)
 
diff --git a/ossfuzz/compress_hc_fuzzer.c b/ossfuzz/compress_hc_fuzzer.c
new file mode 100644
index 0000000..4841367
--- /dev/null
+++ b/ossfuzz/compress_hc_fuzzer.c
@@ -0,0 +1,57 @@
+/**
+ * This fuzz target attempts to compress the fuzzed data with the simple
+ * compression function with an output buffer that may be too small to
+ * ensure that the compressor never crashes.
+ */
+
+#include 
+#include 
+#include 
+#include 
+
+#include "fuzz_helpers.h"
+#include "lz4.h"
+#include "lz4hc.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
+{
+    uint32_t seed = FUZZ_seed(&data, &size);
+    size_t const dstCapacity = FUZZ_rand32(&seed, 0, LZ4_compressBound(size));
+    char* const dst = (char*)malloc(dstCapacity);
+    char* const rt = (char*)malloc(size);
+    int const level = FUZZ_rand32(&seed, LZ4HC_CLEVEL_MIN, LZ4HC_CLEVEL_MAX);
+
+    FUZZ_ASSERT(dst);
+    FUZZ_ASSERT(rt);
+
+    /* If compression succeeds it must round trip correctly. */
+    {
+        int const dstSize = LZ4_compress_HC((const char*)data, dst, size,
+                                            dstCapacity, level);
+        if (dstSize > 0) {
+            int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size);
+            FUZZ_ASSERT_MSG(rtSize == size, "Incorrect regenerated size");
+            FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!");
+        }
+    }
+
+    if (dstCapacity > 0) {
+        /* Compression succeeds and must round trip correctly. */
+        void* state = malloc(LZ4_sizeofStateHC());
+        FUZZ_ASSERT(state);
+        int compressedSize = size;
+        int const dstSize = LZ4_compress_HC_destSize(state, (const char*)data,
+                                                     dst, &compressedSize,
+                                                     dstCapacity, level);
+        FUZZ_ASSERT(dstSize > 0);
+        int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size);
+        FUZZ_ASSERT_MSG(rtSize == compressedSize, "Incorrect regenerated size");
+        FUZZ_ASSERT_MSG(!memcmp(data, rt, compressedSize), "Corruption!");
+        free(state);
+    }
+
+    free(dst);
+    free(rt);
+
+    return 0;
+}
diff --git a/ossfuzz/fuzz_helpers.h b/ossfuzz/fuzz_helpers.h
index 626f209..c4a8645 100644
--- a/ossfuzz/fuzz_helpers.h
+++ b/ossfuzz/fuzz_helpers.h
@@ -24,8 +24,13 @@
 extern "C" {
 #endif
 
-#define MIN(a, b) ((a) < (b) ? (a) : (b))
-#define MAX(a, b) ((a) > (b) ? (a) : (b))
+#define LZ4_COMMONDEFS_ONLY
+#ifndef LZ4_SRC_INCLUDED
+#include "lz4.c"   /* LZ4_count, constants, mem */
+#endif
+
+#define MIN(a,b)   ( (a) < (b) ? (a) : (b) )
+#define MAX(a,b)   ( (a) > (b) ? (a) : (b) )
 
 #define FUZZ_QUOTE_IMPL(str) #str
 #define FUZZ_QUOTE(str) FUZZ_QUOTE_IMPL(str)
diff --git a/ossfuzz/round_trip_hc_fuzzer.c b/ossfuzz/round_trip_hc_fuzzer.c
new file mode 100644
index 0000000..325cdf0
--- /dev/null
+++ b/ossfuzz/round_trip_hc_fuzzer.c
@@ -0,0 +1,39 @@
+/**
+ * This fuzz target performs a lz4 round-trip test (compress & decompress),
+ * compares the result with the original, and calls abort() on corruption.
+ */
+
+#include 
+#include 
+#include 
+#include 
+
+#include "fuzz_helpers.h"
+#include "lz4.h"
+#include "lz4hc.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
+{
+    uint32_t seed = FUZZ_seed(&data, &size);
+    size_t const dstCapacity = LZ4_compressBound(size);
+    char* const dst = (char*)malloc(dstCapacity);
+    char* const rt = (char*)malloc(size);
+    int const level = FUZZ_rand32(&seed, LZ4HC_CLEVEL_MIN, LZ4HC_CLEVEL_MAX);
+
+    FUZZ_ASSERT(dst);
+    FUZZ_ASSERT(rt);
+
+    /* Compression must succeed and round trip correctly. */
+    int const dstSize = LZ4_compress_HC((const char*)data, dst, size,
+                                        dstCapacity, level);
+    FUZZ_ASSERT(dstSize > 0);
+
+    int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size);
+    FUZZ_ASSERT_MSG(rtSize == size, "Incorrect size");
+    FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!");
+
+    free(dst);
+    free(rt);
+
+    return 0;
+}
diff --git a/ossfuzz/round_trip_stream_fuzzer.c b/ossfuzz/round_trip_stream_fuzzer.c
index 4facfc9..abfcd2d 100644
--- a/ossfuzz/round_trip_stream_fuzzer.c
+++ b/ossfuzz/round_trip_stream_fuzzer.c
@@ -12,6 +12,8 @@
 #include "fuzz_helpers.h"
 #define LZ4_STATIC_LINKING_ONLY
 #include "lz4.h"
+#define LZ4_HC_STATIC_LINKING_ONLY
+#include "lz4hc.h"
 
 typedef struct {
   char const* buf;
@@ -27,11 +29,13 @@ typedef struct {
 
 typedef struct {
   LZ4_stream_t* cstream;
+  LZ4_streamHC_t* cstreamHC;
   LZ4_streamDecode_t* dstream;
   const_cursor_t data;
   cursor_t compressed;
   cursor_t roundTrip;
   uint32_t seed;
+  int level;
 } state_t;
 
 cursor_t cursor_create(size_t size)
@@ -44,6 +48,8 @@ cursor_t cursor_create(size_t size)
   return cursor;
 }
 
+typedef void (*round_trip_t)(state_t* state);
+
 void cursor_free(cursor_t cursor)
 {
     free(cursor.buf);
@@ -65,6 +71,8 @@ state_t state_create(char const* data, size_t size, uint32_t seed)
 
     state.cstream = LZ4_createStream();
     FUZZ_ASSERT(state.cstream);
+    state.cstreamHC = LZ4_createStreamHC();
+    FUZZ_ASSERT(state.cstream);
     state.dstream = LZ4_createStreamDecode();
     FUZZ_ASSERT(state.dstream);
 
@@ -76,12 +84,15 @@ void state_free(state_t state)
     cursor_free(state.compressed);
     cursor_free(state.roundTrip);
     LZ4_freeStream(state.cstream);
+    LZ4_freeStreamHC(state.cstreamHC);
     LZ4_freeStreamDecode(state.dstream);
 }
 
 static void state_reset(state_t* state, uint32_t seed)
 {
+    state->level = FUZZ_rand32(&seed, LZ4HC_CLEVEL_MIN, LZ4HC_CLEVEL_MAX);
     LZ4_resetStream_fast(state->cstream);
+    LZ4_resetStreamHC_fast(state->cstreamHC, state->level);
     LZ4_setStreamDecode(state->dstream, NULL, 0);
     state->data.pos = 0;
     state->compressed.pos = 0;
@@ -109,14 +120,19 @@ static void state_checkRoundTrip(state_t const* state)
 
 /**
  * Picks a dictionary size and trims the dictionary off of the data.
+ * We copy the dictionary to the roundTrip so our validation passes.
  */
 static size_t state_trimDict(state_t* state)
 {
     /* 64 KB is the max dict size, allow slightly beyond that to test trim. */
     uint32_t maxDictSize = MIN(70 * 1024, state->data.size);
     size_t const dictSize = FUZZ_rand32(&state->seed, 0, maxDictSize);
-    FUZZ_ASSERT(state->compressed.pos == 0);
-    state->compressed.pos += dictSize;
+    DEBUGLOG(2, "dictSize = %zu", dictSize);
+    FUZZ_ASSERT(state->data.pos == 0);
+    FUZZ_ASSERT(state->roundTrip.pos == 0);
+    memcpy(state->roundTrip.buf, state->data.buf, dictSize);
+    state->data.pos += dictSize;
+    state->roundTrip.pos += dictSize;
     return dictSize;
 }
 
@@ -159,43 +175,111 @@ static void state_extDictRoundTrip(state_t* state)
     cursor_free(data2);
 }
 
-static void state_randomRoundTrip(state_t* state)
+static void state_randomRoundTrip(state_t* state, round_trip_t rt0,
+                                  round_trip_t rt1)
 {
     if (FUZZ_rand32(&state->seed, 0, 1)) {
-      state_prefixRoundTrip(state);
+      rt0(state);
     } else {
-      state_extDictRoundTrip(state);
+      rt1(state);
     }
 }
 
 static void state_loadDictRoundTrip(state_t* state)
 {
-    char const* dict = state->compressed.buf;
+    char const* dict = state->data.buf;
     size_t const dictSize = state_trimDict(state);
     LZ4_loadDict(state->cstream, dict, dictSize);
     LZ4_setStreamDecode(state->dstream, dict, dictSize);
-    state_randomRoundTrip(state);
+    state_randomRoundTrip(state, state_prefixRoundTrip, state_extDictRoundTrip);
 }
 
 static void state_attachDictRoundTrip(state_t* state)
 {
-    char const* dict = state->compressed.buf;
+    char const* dict = state->data.buf;
     size_t const dictSize = state_trimDict(state);
     LZ4_stream_t* dictStream = LZ4_createStream();
     LZ4_loadDict(dictStream, dict, dictSize);
     LZ4_attach_dictionary(state->cstream, dictStream);
     LZ4_setStreamDecode(state->dstream, dict, dictSize);
-    state_randomRoundTrip(state);
+    state_randomRoundTrip(state, state_prefixRoundTrip, state_extDictRoundTrip);
     LZ4_freeStream(dictStream);
 }
 
-typedef void (*round_trip_t)(state_t* state);
+static void state_prefixHCRoundTrip(state_t* state)
+{
+    while (state->data.pos != state->data.size) {
+        char const* src = state->data.buf + state->data.pos;
+        char* dst = state->compressed.buf + state->compressed.pos;
+        int const srcRemaining = state->data.size - state->data.pos;
+        int const srcSize = FUZZ_rand32(&state->seed, 0, srcRemaining);
+        int const dstCapacity = state->compressed.size - state->compressed.pos;
+        int const cSize = LZ4_compress_HC_continue(state->cstreamHC, src, dst,
+                                                   srcSize, dstCapacity);
+        FUZZ_ASSERT(cSize > 0);
+        state->data.pos += srcSize;
+        state->compressed.pos += cSize;
+        state_decompress(state, dst, cSize);
+    }
+}
+
+static void state_extDictHCRoundTrip(state_t* state)
+{
+    int i = 0;
+    cursor_t data2 = cursor_create(state->data.size);
+    DEBUGLOG(2, "extDictHC");
+    memcpy(data2.buf, state->data.buf, state->data.size);
+    while (state->data.pos != state->data.size) {
+        char const* data = (i++ & 1) ? state->data.buf : data2.buf;
+        char const* src = data + state->data.pos;
+        char* dst = state->compressed.buf + state->compressed.pos;
+        int const srcRemaining = state->data.size - state->data.pos;
+        int const srcSize = FUZZ_rand32(&state->seed, 0, srcRemaining);
+        int const dstCapacity = state->compressed.size - state->compressed.pos;
+        int const cSize = LZ4_compress_HC_continue(state->cstreamHC, src, dst,
+                                                   srcSize, dstCapacity);
+        FUZZ_ASSERT(cSize > 0);
+        DEBUGLOG(2, "srcSize = %d", srcSize);
+        state->data.pos += srcSize;
+        state->compressed.pos += cSize;
+        state_decompress(state, dst, cSize);
+    }
+    cursor_free(data2);
+}
+
+static void state_loadDictHCRoundTrip(state_t* state)
+{
+    char const* dict = state->data.buf;
+    size_t const dictSize = state_trimDict(state);
+    LZ4_loadDictHC(state->cstreamHC, dict, dictSize);
+    LZ4_setStreamDecode(state->dstream, dict, dictSize);
+    state_randomRoundTrip(state, state_prefixHCRoundTrip,
+                          state_extDictHCRoundTrip);
+}
+
+static void state_attachDictHCRoundTrip(state_t* state)
+{
+    char const* dict = state->data.buf;
+    size_t const dictSize = state_trimDict(state);
+    LZ4_streamHC_t* dictStream = LZ4_createStreamHC();
+    LZ4_setCompressionLevel(dictStream, state->level);
+    LZ4_loadDictHC(dictStream, dict, dictSize);
+    LZ4_attach_HC_dictionary(state->cstreamHC, dictStream);
+    LZ4_setStreamDecode(state->dstream, dict, dictSize);
+    state_randomRoundTrip(state, state_prefixHCRoundTrip,
+                          state_extDictHCRoundTrip);
+    LZ4_freeStreamHC(dictStream);
+}
 
 round_trip_t roundTrips[] = {
   &state_prefixRoundTrip,
   &state_extDictRoundTrip,
   &state_loadDictRoundTrip,
   &state_attachDictRoundTrip,
+  &state_prefixHCRoundTrip,
+  &state_extDictHCRoundTrip,
+  &state_loadDictHCRoundTrip,
+  &state_attachDictHCRoundTrip,
 };
 
 int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
@@ -206,9 +290,10 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
     int i;
 
     for (i = 0; i < n; ++i) {
+        DEBUGLOG(2, "Round trip %d", i);
+        state_reset(&state, seed);
         roundTrips[i](&state);
         state_checkRoundTrip(&state);
-        state_reset(&state, seed);
     }
 
     state_free(state);
-- 
cgit v0.12


From 1f236e0790ab28a223e89bbb8e65d3ed067adc89 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 17 Jul 2019 19:40:04 -0700
Subject: Fix LZ4_attach_dictionary with empty dictionary

---
 lib/lz4.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 74a9247..81b3c10 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1429,7 +1429,7 @@ void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dict
      */
     LZ4_resetStream_fast(workingStream);
 
-    if (dictionaryStream != NULL) {
+    if (dictionaryStream != NULL && dictionaryStream->internal_donotuse.dictSize > 0) {
         /* If the current offset is zero, we will never look in the
          * external dictionary context, since there is no value a table
          * entry can take that indicate a miss. In that case, we need
-- 
cgit v0.12


From b487660309d4245eec87e3ada4712bc2a19df791 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Thu, 18 Jul 2019 18:45:32 -0700
Subject: [lz4frame] Skip magic and checksums in fuzzing mode

When `FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION` is defined we skip
magic and checksum checks. This makes it easier to fuzz decompression.
---
 lib/lz4frame.c | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index cc7f2d5..13c1ae9 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -1131,8 +1131,10 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
     }
 
     /* control magic number */
+#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
     if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER)
         return err0r(LZ4F_ERROR_frameType_unknown);
+#endif
     dctx->frameInfo.frameType = LZ4F_frame;
 
     /* Flags */
@@ -1171,10 +1173,12 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
 
     /* check header */
     assert(frameHeaderSize > 5);
+#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
     {   BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5);
         if (HC != srcPtr[frameHeaderSize-1])
             return err0r(LZ4F_ERROR_headerChecksum_invalid);
     }
+#endif
 
     /* save */
     dctx->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode;
@@ -1211,8 +1215,10 @@ size_t LZ4F_headerSize(const void* src, size_t srcSize)
         return 8;
 
     /* control magic number */
+#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
     if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER)
         return err0r(LZ4F_ERROR_frameType_unknown);
+#endif
 
     /* Frame Header Size */
     {   BYTE const FLG = ((const BYTE*)src)[4];
@@ -1555,8 +1561,10 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
                 }
                 {   U32 const readCRC = LZ4F_readLE32(crcSrc);
                     U32 const calcCRC = XXH32_digest(&dctx->blockChecksum);
+#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
                     if (readCRC != calcCRC)
                         return err0r(LZ4F_ERROR_blockChecksum_invalid);
+#endif
             }   }
             dctx->dStage = dstage_getBlockHeader;  /* new block */
             break;
@@ -1595,8 +1603,10 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
                 assert(selectedIn != NULL);  /* selectedIn is defined at this stage (either srcPtr, or dctx->tmpIn) */
                 {   U32 const readBlockCrc = LZ4F_readLE32(selectedIn + dctx->tmpInTarget);
                     U32 const calcBlockCrc = XXH32(selectedIn, dctx->tmpInTarget, 0);
+#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
                     if (readBlockCrc != calcBlockCrc)
                         return err0r(LZ4F_ERROR_blockChecksum_invalid);
+#endif
             }   }
 
             if ((size_t)(dstEnd-dstPtr) >= dctx->maxBlockSize) {
@@ -1724,8 +1734,10 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
         /* case dstage_checkSuffix: */   /* no direct entry, avoid initialization risks */
             {   U32 const readCRC = LZ4F_readLE32(selectedIn);
                 U32 const resultCRC = XXH32_digest(&(dctx->xxh));
+#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
                 if (readCRC != resultCRC)
                     return err0r(LZ4F_ERROR_contentChecksum_invalid);
+#endif
                 nextSrcSizeHint = 0;
                 LZ4F_resetDecompressionContext(dctx);
                 doAnotherStage = 0;
-- 
cgit v0.12


From d28159c025829fc70a295b727e32f899a9e0c7c5 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Thu, 18 Jul 2019 18:49:40 -0700
Subject: [fuzz] Add LZ4 frame fuzzers

* Round trip fuzzer
* Compress fuzzer
* Decompress fuzzer
---
 ossfuzz/Makefile                  | 10 ++++--
 ossfuzz/compress_frame_fuzzer.c   | 42 ++++++++++++++++++++++++
 ossfuzz/decompress_frame_fuzzer.c | 67 +++++++++++++++++++++++++++++++++++++++
 ossfuzz/lz4_helpers.c             | 51 +++++++++++++++++++++++++++++
 ossfuzz/lz4_helpers.h             | 13 ++++++++
 ossfuzz/round_trip_frame_fuzzer.c | 39 +++++++++++++++++++++++
 6 files changed, 219 insertions(+), 3 deletions(-)
 create mode 100644 ossfuzz/compress_frame_fuzzer.c
 create mode 100644 ossfuzz/decompress_frame_fuzzer.c
 create mode 100644 ossfuzz/lz4_helpers.c
 create mode 100644 ossfuzz/lz4_helpers.h
 create mode 100644 ossfuzz/round_trip_frame_fuzzer.c

diff --git a/ossfuzz/Makefile b/ossfuzz/Makefile
index 9974b81..6875eb6 100644
--- a/ossfuzz/Makefile
+++ b/ossfuzz/Makefile
@@ -33,7 +33,8 @@ DEBUGFLAGS = -g -DLZ4_DEBUG=$(DEBUGLEVEL)
 
 LZ4_CFLAGS  = $(CFLAGS) $(DEBUGFLAGS) $(MOREFLAGS)
 LZ4_CXXFLAGS = $(CXXFLAGS) $(DEBUGFLAGS) $(MOREFLAGS)
-LZ4_CPPFLAGS = $(CPPFLAGS) -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_
+LZ4_CPPFLAGS = $(CPPFLAGS) -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_ \
+               -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
 
 FUZZERS := \
 	compress_fuzzer \
@@ -41,7 +42,10 @@ FUZZERS := \
 	round_trip_fuzzer \
 	round_trip_stream_fuzzer \
 	compress_hc_fuzzer \
-	round_trip_hc_fuzzer
+	round_trip_hc_fuzzer \
+	compress_frame_fuzzer \
+	round_trip_frame_fuzzer \
+	decompress_frame_fuzzer
 
 all: $(FUZZERS)
 
@@ -54,7 +58,7 @@ $(LZ4DIR)/liblz4.a:
 	$(CC) -c $(LZ4_CFLAGS) $(LZ4_CPPFLAGS) $< -o $@
 
 # Generic rule for generating fuzzers
-%_fuzzer: %_fuzzer.o $(LZ4DIR)/liblz4.a
+%_fuzzer: %_fuzzer.o lz4_helpers.o $(LZ4DIR)/liblz4.a
 	# Compile the standalone code just in case. The OSS-Fuzz code might
 	# override the LIB_FUZZING_ENGINE value to "-fsanitize=fuzzer"
 	$(CC) -c $(LZ4_CFLAGS) $(LZ4_CPPFLAGS) standaloneengine.c -o standaloneengine.o
diff --git a/ossfuzz/compress_frame_fuzzer.c b/ossfuzz/compress_frame_fuzzer.c
new file mode 100644
index 0000000..75c609f
--- /dev/null
+++ b/ossfuzz/compress_frame_fuzzer.c
@@ -0,0 +1,42 @@
+/**
+ * This fuzz target attempts to compress the fuzzed data with the simple
+ * compression function with an output buffer that may be too small to
+ * ensure that the compressor never crashes.
+ */
+
+#include 
+#include 
+#include 
+#include 
+
+#include "fuzz_helpers.h"
+#include "lz4.h"
+#include "lz4frame.h"
+#include "lz4_helpers.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
+{
+    uint32_t seed = FUZZ_seed(&data, &size);
+    LZ4F_preferences_t const prefs = FUZZ_randomPreferences(&seed);
+    size_t const compressBound = LZ4F_compressFrameBound(size, &prefs);
+    size_t const dstCapacity = FUZZ_rand32(&seed, 0, compressBound);
+    char* const dst = (char*)malloc(dstCapacity);
+    char* const rt = (char*)malloc(size);
+
+    FUZZ_ASSERT(dst);
+    FUZZ_ASSERT(rt);
+
+    /* If compression succeeds it must round trip correctly. */
+    size_t const dstSize =
+            LZ4F_compressFrame(dst, dstCapacity, data, size, &prefs);
+    if (!LZ4F_isError(dstSize)) {
+        size_t const rtSize = FUZZ_decompressFrame(rt, size, dst, dstSize);
+        FUZZ_ASSERT_MSG(rtSize == size, "Incorrect regenerated size");
+        FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!");
+    }
+
+    free(dst);
+    free(rt);
+
+    return 0;
+}
diff --git a/ossfuzz/decompress_frame_fuzzer.c b/ossfuzz/decompress_frame_fuzzer.c
new file mode 100644
index 0000000..bda25b0
--- /dev/null
+++ b/ossfuzz/decompress_frame_fuzzer.c
@@ -0,0 +1,67 @@
+/**
+ * This fuzz target attempts to decompress the fuzzed data with the simple
+ * decompression function to ensure the decompressor never crashes.
+ */
+
+#include 
+#include 
+#include 
+#include 
+
+#include "fuzz_helpers.h"
+#include "lz4.h"
+#define LZ4F_STATIC_LINKING_ONLY
+#include "lz4frame.h"
+#include "lz4_helpers.h"
+
+static void decompress(LZ4F_dctx* dctx, void* dst, size_t dstCapacity,
+                       const void* src, size_t srcSize,
+                       const void* dict, size_t dictSize,
+                       const LZ4F_decompressOptions_t* opts)
+{
+    LZ4F_resetDecompressionContext(dctx);
+    if (dictSize == 0)
+        LZ4F_decompress(dctx, dst, &dstCapacity, src, &srcSize, opts);
+    else
+        LZ4F_decompress_usingDict(dctx, dst, &dstCapacity, src, &srcSize,
+                                  dict, dictSize, opts);
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
+{
+
+    uint32_t seed = FUZZ_seed(&data, &size);
+    size_t const dstCapacity = FUZZ_rand32(&seed, 0, 4 * size);
+    size_t const largeDictSize = 64 * 1024;
+    size_t const dictSize = FUZZ_rand32(&seed, 0, largeDictSize);
+    char* const dst = (char*)malloc(dstCapacity);
+    char* const dict = (char*)malloc(dictSize);
+    LZ4F_decompressOptions_t opts;
+    LZ4F_dctx* dctx;
+    LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
+
+    FUZZ_ASSERT(dctx);
+    FUZZ_ASSERT(dst);
+    FUZZ_ASSERT(dict);
+
+    /* Prepare the dictionary. The data doesn't matter for decompression. */
+    memset(dict, 0, dictSize);
+
+
+    /* Decompress using multiple configurations. */
+    memset(&opts, 0, sizeof(opts));
+    opts.stableDst = 0;
+    decompress(dctx, dst, dstCapacity, data, size, NULL, 0, &opts);
+    opts.stableDst = 1;
+    decompress(dctx, dst, dstCapacity, data, size, NULL, 0, &opts);
+    opts.stableDst = 0;
+    decompress(dctx, dst, dstCapacity, data, size, dict, dictSize, &opts);
+    opts.stableDst = 1;
+    decompress(dctx, dst, dstCapacity, data, size, dict, dictSize, &opts);
+
+    LZ4F_freeDecompressionContext(dctx);
+    free(dst);
+    free(dict);
+
+    return 0;
+}
diff --git a/ossfuzz/lz4_helpers.c b/ossfuzz/lz4_helpers.c
new file mode 100644
index 0000000..9471630
--- /dev/null
+++ b/ossfuzz/lz4_helpers.c
@@ -0,0 +1,51 @@
+#include "fuzz_helpers.h"
+#include "lz4_helpers.h"
+#include "lz4hc.h"
+
+LZ4F_frameInfo_t FUZZ_randomFrameInfo(uint32_t* seed)
+{
+    LZ4F_frameInfo_t info = LZ4F_INIT_FRAMEINFO;
+    info.blockSizeID = FUZZ_rand32(seed, LZ4F_max64KB - 1, LZ4F_max4MB);
+    if (info.blockSizeID < LZ4F_max64KB) {
+        info.blockSizeID = LZ4F_default;
+    }
+    info.blockMode = FUZZ_rand32(seed, LZ4F_blockLinked, LZ4F_blockIndependent);
+    info.contentChecksumFlag = FUZZ_rand32(seed, LZ4F_noContentChecksum,
+                                           LZ4F_contentChecksumEnabled);
+    info.blockChecksumFlag = FUZZ_rand32(seed, LZ4F_noBlockChecksum,
+                                         LZ4F_blockChecksumEnabled);
+    return info;
+}
+
+LZ4F_preferences_t FUZZ_randomPreferences(uint32_t* seed)
+{
+    LZ4F_preferences_t prefs = LZ4F_INIT_PREFERENCES;
+    prefs.frameInfo = FUZZ_randomFrameInfo(seed);
+    prefs.compressionLevel = FUZZ_rand32(seed, 0, LZ4HC_CLEVEL_MAX + 3) - 3;
+    prefs.autoFlush = FUZZ_rand32(seed, 0, 1);
+    prefs.favorDecSpeed = FUZZ_rand32(seed, 0, 1);
+    return prefs;
+}
+
+size_t FUZZ_decompressFrame(void* dst, const size_t dstCapacity,
+                            const void* src, const size_t srcSize)
+{
+    LZ4F_decompressOptions_t opts;
+    memset(&opts, 0, sizeof(opts));
+    opts.stableDst = 1;
+    LZ4F_dctx* dctx;
+    LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
+    FUZZ_ASSERT(dctx);
+
+    size_t dstSize = dstCapacity;
+    size_t srcConsumed = srcSize;
+    size_t const rc =
+            LZ4F_decompress(dctx, dst, &dstSize, src, &srcConsumed, &opts);
+    FUZZ_ASSERT(!LZ4F_isError(rc));
+    FUZZ_ASSERT(rc == 0);
+    FUZZ_ASSERT(srcConsumed == srcSize);
+
+    LZ4F_freeDecompressionContext(dctx);
+
+    return dstSize;
+}
diff --git a/ossfuzz/lz4_helpers.h b/ossfuzz/lz4_helpers.h
new file mode 100644
index 0000000..c99fb01
--- /dev/null
+++ b/ossfuzz/lz4_helpers.h
@@ -0,0 +1,13 @@
+#ifndef LZ4_HELPERS
+#define LZ4_HELPERS
+
+#include "lz4frame.h"
+
+LZ4F_frameInfo_t FUZZ_randomFrameInfo(uint32_t* seed);
+
+LZ4F_preferences_t FUZZ_randomPreferences(uint32_t* seed);
+
+size_t FUZZ_decompressFrame(void* dst, const size_t dstCapacity,
+                            const void* src, const size_t srcSize);
+
+#endif /* LZ4_HELPERS */
diff --git a/ossfuzz/round_trip_frame_fuzzer.c b/ossfuzz/round_trip_frame_fuzzer.c
new file mode 100644
index 0000000..1eea90c
--- /dev/null
+++ b/ossfuzz/round_trip_frame_fuzzer.c
@@ -0,0 +1,39 @@
+/**
+ * This fuzz target performs a lz4 round-trip test (compress & decompress),
+ * compares the result with the original, and calls abort() on corruption.
+ */
+
+#include 
+#include 
+#include 
+#include 
+
+#include "fuzz_helpers.h"
+#include "lz4.h"
+#include "lz4frame.h"
+#include "lz4_helpers.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
+{
+    uint32_t seed = FUZZ_seed(&data, &size);
+    LZ4F_preferences_t const prefs = FUZZ_randomPreferences(&seed);
+    size_t const dstCapacity = LZ4F_compressFrameBound(size, &prefs);
+    char* const dst = (char*)malloc(dstCapacity);
+    char* const rt = (char*)malloc(size);
+
+    FUZZ_ASSERT(dst);
+    FUZZ_ASSERT(rt);
+
+    /* Compression must succeed and round trip correctly. */
+    size_t const dstSize =
+            LZ4F_compressFrame(dst, dstCapacity, data, size, &prefs);
+    FUZZ_ASSERT(!LZ4F_isError(dstSize));
+    size_t const rtSize = FUZZ_decompressFrame(rt, size, dst, dstSize);
+    FUZZ_ASSERT_MSG(rtSize == size, "Incorrect regenerated size");
+    FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!");
+
+    free(dst);
+    free(rt);
+
+    return 0;
+}
-- 
cgit v0.12


From 87e52f7d5d8e2e44c107ca266d91a5ff669325b1 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 19 Jul 2019 14:44:06 -0700
Subject: [lz4frame] Fix unused variable warnings in fuzzing mode

---
 lib/lz4frame.c | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 13c1ae9..c9f630d 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -1564,6 +1564,9 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
 #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
                     if (readCRC != calcCRC)
                         return err0r(LZ4F_ERROR_blockChecksum_invalid);
+#else
+                    (void)readCRC;
+                    (void)calcCRC;
 #endif
             }   }
             dctx->dStage = dstage_getBlockHeader;  /* new block */
@@ -1606,6 +1609,9 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
 #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
                     if (readBlockCrc != calcBlockCrc)
                         return err0r(LZ4F_ERROR_blockChecksum_invalid);
+#else
+                    (void)readBlockCrc;
+                    (void)calcBlockCrc;
 #endif
             }   }
 
@@ -1737,6 +1743,9 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
 #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
                 if (readCRC != resultCRC)
                     return err0r(LZ4F_ERROR_contentChecksum_invalid);
+#else
+                (void)readCRC;
+                (void)resultCRC;
 #endif
                 nextSrcSizeHint = 0;
                 LZ4F_resetDecompressionContext(dctx);
-- 
cgit v0.12


From eee8cc79e79230a884884beb885ea92fe1b70928 Mon Sep 17 00:00:00 2001
From: WHR 
Date: Tue, 23 Jul 2019 13:37:11 +0800
Subject: lz4cli: add option '--best' as an alias of '-12'

---
 programs/lz4.1    | 4 ++++
 programs/lz4.1.md | 3 +++
 programs/lz4cli.c | 4 ++++
 3 files changed, 11 insertions(+)

diff --git a/programs/lz4.1 b/programs/lz4.1
index 3ca017a..d758ed5 100644
--- a/programs/lz4.1
+++ b/programs/lz4.1
@@ -120,6 +120,10 @@ Compression level, with # being any value from 1 to 12\. Higher values trade com
 Switch to ultra\-fast compression levels\. The higher the value, the faster the compression speed, at the cost of some compression ratio\. If \fB=#\fR is not present, it defaults to \fB1\fR\. This setting overrides compression level if one was set previously\. Similarly, if a compression level is set after \fB\-\-fast\fR, it overrides it\.
 .
 .TP
+\fB\-\-best\fR
+Set highest compression level\. Same as -12\.
+.
+.TP
 \fB\-\-favor\-decSpeed\fR
 Generate compressed data optimized for decompression speed\. Compressed data will be larger as a consequence (typically by ~0\.5%), while decompression speed will be improved by 5\-20%, depending on use cases\. This option only works in combination with very high compression levels (>=10)\.
 .
diff --git a/programs/lz4.1.md b/programs/lz4.1.md
index 8874467..56c0053 100644
--- a/programs/lz4.1.md
+++ b/programs/lz4.1.md
@@ -135,6 +135,9 @@ only the latest one will be applied.
   This setting overrides compression level if one was set previously.
   Similarly, if a compression level is set after `--fast`, it overrides it.
 
+* `--best`:
+  Set highest compression level. Same as -12.
+
 * `--favor-decSpeed`:
   Generate compressed data optimized for decompression speed.
   Compressed data will be larger as a consequence (typically by ~0.5%),
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index 3b5d61e..5da7654 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -146,6 +146,7 @@ static int usage_advanced(const char* exeName)
     DISPLAY( "--[no-]sparse  : sparse mode (default:enabled on file, disabled on stdout)\n");
     DISPLAY( "--favor-decSpeed: compressed files decompress faster, but are less compressed \n");
     DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %i)\n", 1);
+    DISPLAY( "--best  : same as -%d\n", LZ4HC_CLEVEL_MAX);
     DISPLAY( "Benchmark arguments : \n");
     DISPLAY( " -b#    : benchmark file(s), using # compression level (default : 1) \n");
     DISPLAY( " -e#    : test all compression levels from -bX to # (default : 1)\n");
@@ -414,6 +415,9 @@ int main(int argc, const char** argv)
                         }
                         continue;
                     }
+
+                /* For gzip(1) compatibility */
+                if (!strcmp(argument,  "--best")) { cLevel=LZ4HC_CLEVEL_MAX; continue; }
             }
 
             while (argument[1]!=0) {
-- 
cgit v0.12


From 4c1d4c437daddc36baf0f4616f2392f67ac320d4 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Tue, 23 Jul 2019 17:54:09 -0700
Subject: [LZ4HC] Speed up pattern compression with external dictionary

Fixes #761.
---
 lib/lz4hc.c | 74 +++++++++++++++++++++++++++++++++++++++----------------------
 1 file changed, 47 insertions(+), 27 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 29288a5..b8ff72a 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -151,6 +151,19 @@ int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match,
     return back;
 }
 
+#if defined(_MSC_VER)
+#  define LZ4HC_rotl32(x,r) _rotl(x,r)
+#else
+#  define LZ4HC_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+#endif
+
+
+static U32 LZ4HC_rotatePattern(size_t const length, U32 const pattern)
+{
+    size_t const bitsToRotate = (length & (sizeof(pattern) - 1)) << 3;
+    return LZ4HC_rotl32(pattern, (int)bitsToRotate);
+}
+
 /* LZ4HC_countPattern() :
  * pattern32 must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) */
 static unsigned
@@ -313,34 +326,41 @@ LZ4HC_InsertAndGetWiderMatch (
                     } else {
                         repeat = rep_not;
                 }   }
-                if ( (repeat == rep_confirmed)
-                  && (matchCandidateIdx >= dictLimit) ) {   /* same segment only */
-                    const BYTE* const matchPtr = base + matchCandidateIdx;
+                if ( (repeat == rep_confirmed) && (matchCandidateIdx >= lowestMatchIndex) ) {
+                    const int extDict = matchCandidateIdx < dictLimit;
+                    const BYTE* const matchPtr = (extDict ? dictBase : base) + matchCandidateIdx;
                     if (LZ4_read32(matchPtr) == pattern) {  /* good candidate */
-                        size_t const forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern);
-                        const BYTE* const lowestMatchPtr = (lowPrefixPtr + LZ4_DISTANCE_MAX >= ip) ? lowPrefixPtr : ip - LZ4_DISTANCE_MAX;
-                        size_t const backLength = LZ4HC_reverseCountPattern(matchPtr, lowestMatchPtr, pattern);
-                        size_t const currentSegmentLength = backLength + forwardPatternLength;
-
-                        if ( (currentSegmentLength >= srcPatternLength)   /* current pattern segment large enough to contain full srcPatternLength */
-                          && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */
-                            matchIndex = matchCandidateIdx + (U32)forwardPatternLength - (U32)srcPatternLength;  /* best position, full pattern, might be followed by more match */
-                        } else {
-                            matchIndex = matchCandidateIdx - (U32)backLength;   /* farthest position in current segment, will find a match of length currentSegmentLength + maybe some back */
-                            if (lookBackLength==0) {  /* no back possible */
-                                size_t const maxML = MIN(currentSegmentLength, srcPatternLength);
-                                if ((size_t)longest < maxML) {
-                                    assert(base + matchIndex < ip);
-                                    if (ip - (base+matchIndex) > LZ4_DISTANCE_MAX) break;
-                                    assert(maxML < 2 GB);
-                                    longest = (int)maxML;
-                                    *matchpos = base + matchIndex;   /* virtual pos, relative to ip, to retrieve offset */
-                                    *startpos = ip;
-                                }
-                                {   U32 const distToNextPattern = DELTANEXTU16(chainTable, matchIndex);
-                                    if (distToNextPattern > matchIndex) break;  /* avoid overflow */
-                                    matchIndex -= distToNextPattern;
-                        }   }   }
+                        const BYTE* const dictStart = dictBase + hc4->lowLimit;
+                        const BYTE* const iLimit = extDict ? dictBase + dictLimit : iHighLimit;
+                        size_t forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iLimit, pattern) + sizeof(pattern);
+                        if (extDict && ip + forwardPatternLength == iLimit) {
+                            U32 const rotatedPattern = LZ4HC_rotatePattern(forwardPatternLength, pattern);
+                            forwardPatternLength += LZ4HC_countPattern(lowPrefixPtr, iHighLimit, rotatedPattern);
+                        }
+                        {   const BYTE* const lowestMatchPtr = extDict ? dictStart : lowPrefixPtr;
+                            size_t const backLengthRaw = LZ4HC_reverseCountPattern(matchPtr, lowestMatchPtr, pattern);
+                            size_t const backLength = matchCandidateIdx - MAX(matchCandidateIdx - (U32)backLengthRaw, lowestMatchIndex);
+                            size_t const currentSegmentLength = backLength + forwardPatternLength;
+
+                            if ( (currentSegmentLength >= srcPatternLength)   /* current pattern segment large enough to contain full srcPatternLength */
+                              && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */
+                                matchIndex = matchCandidateIdx + (U32)forwardPatternLength - (U32)srcPatternLength;  /* best position, full pattern, might be followed by more match */
+                            } else {
+                                matchIndex = matchCandidateIdx - (U32)backLength;   /* farthest position in current segment, will find a match of length currentSegmentLength + maybe some back */
+                                if (lookBackLength==0) {  /* no back possible */
+                                    size_t const maxML = MIN(currentSegmentLength, srcPatternLength);
+                                    if ((size_t)longest < maxML) {
+                                        assert(base + matchIndex < ip);
+                                        if (ip - (base+matchIndex) > LZ4_DISTANCE_MAX) break;
+                                        assert(maxML < 2 GB);
+                                        longest = (int)maxML;
+                                        *matchpos = base + matchIndex;   /* virtual pos, relative to ip, to retrieve offset */
+                                        *startpos = ip;
+                                    }
+                                    {   U32 const distToNextPattern = DELTANEXTU16(chainTable, matchIndex);
+                                        if (distToNextPattern > matchIndex) break;  /* avoid overflow */
+                                        matchIndex -= distToNextPattern;
+                        }   }   }   }
                         continue;
                 }   }
         }   }   /* PA optimization */
-- 
cgit v0.12


From 7e97bf377d3c3a6d0d71a3ce9116e1f0143a8740 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Tue, 30 Jul 2019 15:16:35 -0700
Subject: [lz4hc] Improve pattern detection in ext dict

It is important to continue to look backwards if the current pattern
reaches `lowPrefixPtr`. If the pattern detection doesn't go all the
way to the beginning of the pattern, or the end of the pattern it
slows down the search instead of speeding it up.

The slow unit in `round_trip_stream_fuzzer` used to take 12 seconds
to run with -O3, now it takes 0.2 seconds.

Credit to OSS-Fuzz
---
 lib/lz4hc.c | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index b8ff72a..4122af4 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -158,9 +158,11 @@ int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match,
 #endif
 
 
-static U32 LZ4HC_rotatePattern(size_t const length, U32 const pattern)
+static U32 LZ4HC_rotatePattern(size_t const rotate, U32 const pattern)
 {
-    size_t const bitsToRotate = (length & (sizeof(pattern) - 1)) << 3;
+    size_t const bitsToRotate = (rotate & (sizeof(pattern) - 1)) << 3;
+    if (bitsToRotate == 0)
+        return pattern;
     return LZ4HC_rotl32(pattern, (int)bitsToRotate);
 }
 
@@ -338,9 +340,15 @@ LZ4HC_InsertAndGetWiderMatch (
                             forwardPatternLength += LZ4HC_countPattern(lowPrefixPtr, iHighLimit, rotatedPattern);
                         }
                         {   const BYTE* const lowestMatchPtr = extDict ? dictStart : lowPrefixPtr;
-                            size_t const backLengthRaw = LZ4HC_reverseCountPattern(matchPtr, lowestMatchPtr, pattern);
-                            size_t const backLength = matchCandidateIdx - MAX(matchCandidateIdx - (U32)backLengthRaw, lowestMatchIndex);
-                            size_t const currentSegmentLength = backLength + forwardPatternLength;
+                            size_t backLength = LZ4HC_reverseCountPattern(matchPtr, lowestMatchPtr, pattern);
+                            size_t currentSegmentLength;
+                            if (!extDict && matchPtr - backLength == lowPrefixPtr && hc4->lowLimit < dictLimit) {
+                                U32 const rotatedPattern = LZ4HC_rotatePattern((U32)(-(int)backLength), pattern);
+                                backLength += LZ4HC_reverseCountPattern(dictBase + dictLimit, dictStart, rotatedPattern);
+                            }
+                            /* Limit backLength not go further than lowestMatchIndex */
+                            backLength = matchCandidateIdx - MAX(matchCandidateIdx - (U32)backLength, lowestMatchIndex);
+                            currentSegmentLength = backLength + forwardPatternLength;
 
                             if ( (currentSegmentLength >= srcPatternLength)   /* current pattern segment large enough to contain full srcPatternLength */
                               && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */
-- 
cgit v0.12


From 58ea585878008db36fa20025703757f0b70836eb Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Tue, 30 Jul 2019 15:21:52 -0700
Subject: [lz4hc] Fix minor pessimization in extDict pattern matching

We should be comparing `matchPtr` not `ip`. This bug just means
that this branch was not taken, so we might miss some of the
forward length.
---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 4122af4..a0b8c48 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -335,7 +335,7 @@ LZ4HC_InsertAndGetWiderMatch (
                         const BYTE* const dictStart = dictBase + hc4->lowLimit;
                         const BYTE* const iLimit = extDict ? dictBase + dictLimit : iHighLimit;
                         size_t forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iLimit, pattern) + sizeof(pattern);
-                        if (extDict && ip + forwardPatternLength == iLimit) {
+                        if (extDict && matchPtr + forwardPatternLength == iLimit) {
                             U32 const rotatedPattern = LZ4HC_rotatePattern(forwardPatternLength, pattern);
                             forwardPatternLength += LZ4HC_countPattern(lowPrefixPtr, iHighLimit, rotatedPattern);
                         }
-- 
cgit v0.12


From be1738aa46326e86e9c3bb1029abaadce45b8e72 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Tue, 30 Jul 2019 23:39:39 -0700
Subject: [lz4hc] Fix pattern detection end of dictionary

The pattern detection in extDict mode could put `matchIndex`
within the last 3 bytes of the dictionary. This would cause
a read out of bounds.
---
 lib/lz4hc.c | 65 ++++++++++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 45 insertions(+), 20 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index a0b8c48..596888a 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -218,6 +218,16 @@ LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern)
     return (unsigned)(iStart - ip);
 }
 
+/* LZ4HC_protectDictEnd() :
+ * Checks if the match is in the last 3 bytes of the dictionary, so reading the
+ * 4 byte MINMATCH would overflow.
+ * @returns true if the match index is okay.
+ */
+static int LZ4HC_protectDictEnd(U32 const dictLimit, U32 const matchIndex)
+{
+    return ((U32)((dictLimit - 1) - matchIndex) >= 3);
+}
+
 typedef enum { rep_untested, rep_not, rep_confirmed } repeat_state_e;
 typedef enum { favorCompressionRatio=0, favorDecompressionSpeed } HCfavor_e;
 
@@ -304,7 +314,7 @@ LZ4HC_InsertAndGetWiderMatch (
             if (matchIndex + (U32)longest <= ipIndex) {
                 U32 distanceToNextMatch = 1;
                 int pos;
-                for (pos = 0; pos <= longest - MINMATCH; pos++) {
+                for (pos = 0; pos <= longest - MINMATCH; ++pos) {
                     U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + (U32)pos);
                     if (candidateDist > distanceToNextMatch) {
                         distanceToNextMatch = candidateDist;
@@ -314,7 +324,8 @@ LZ4HC_InsertAndGetWiderMatch (
                     if (distanceToNextMatch > matchIndex) break;   /* avoid overflow */
                     matchIndex -= distanceToNextMatch;
                     continue;
-        }   }   }
+                }
+        }   }
 
         {   U32 const distNextMatch = DELTANEXTU16(chainTable, matchIndex);
             if (patternAnalysis && distNextMatch==1 && matchChainPos==0) {
@@ -328,7 +339,8 @@ LZ4HC_InsertAndGetWiderMatch (
                     } else {
                         repeat = rep_not;
                 }   }
-                if ( (repeat == rep_confirmed) && (matchCandidateIdx >= lowestMatchIndex) ) {
+                if ( (repeat == rep_confirmed) && (matchCandidateIdx >= lowestMatchIndex)
+                  && LZ4HC_protectDictEnd(dictLimit, matchCandidateIdx) ) {
                     const int extDict = matchCandidateIdx < dictLimit;
                     const BYTE* const matchPtr = (extDict ? dictBase : base) + matchCandidateIdx;
                     if (LZ4_read32(matchPtr) == pattern) {  /* good candidate */
@@ -348,27 +360,40 @@ LZ4HC_InsertAndGetWiderMatch (
                             }
                             /* Limit backLength not go further than lowestMatchIndex */
                             backLength = matchCandidateIdx - MAX(matchCandidateIdx - (U32)backLength, lowestMatchIndex);
+                            assert(matchCandidateIdx - backLength >= lowestMatchIndex);
                             currentSegmentLength = backLength + forwardPatternLength;
-
+                            /* Adjust to end of pattern if the source pattern fits, otherwise the beginning of the pattern */
                             if ( (currentSegmentLength >= srcPatternLength)   /* current pattern segment large enough to contain full srcPatternLength */
                               && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */
-                                matchIndex = matchCandidateIdx + (U32)forwardPatternLength - (U32)srcPatternLength;  /* best position, full pattern, might be followed by more match */
+                                U32 const newMatchIndex = matchCandidateIdx + (U32)forwardPatternLength - (U32)srcPatternLength;  /* best position, full pattern, might be followed by more match */
+                                if (LZ4HC_protectDictEnd(dictLimit, newMatchIndex))
+                                    matchIndex = newMatchIndex;
+                                else {
+                                    /* Can only happen if started in the prefix */
+                                    assert(newMatchIndex >= dictLimit - 3 && newMatchIndex < dictLimit && !extDict);
+                                    matchIndex = dictLimit;
+                                }
                             } else {
-                                matchIndex = matchCandidateIdx - (U32)backLength;   /* farthest position in current segment, will find a match of length currentSegmentLength + maybe some back */
-                                if (lookBackLength==0) {  /* no back possible */
-                                    size_t const maxML = MIN(currentSegmentLength, srcPatternLength);
-                                    if ((size_t)longest < maxML) {
-                                        assert(base + matchIndex < ip);
-                                        if (ip - (base+matchIndex) > LZ4_DISTANCE_MAX) break;
-                                        assert(maxML < 2 GB);
-                                        longest = (int)maxML;
-                                        *matchpos = base + matchIndex;   /* virtual pos, relative to ip, to retrieve offset */
-                                        *startpos = ip;
-                                    }
-                                    {   U32 const distToNextPattern = DELTANEXTU16(chainTable, matchIndex);
-                                        if (distToNextPattern > matchIndex) break;  /* avoid overflow */
-                                        matchIndex -= distToNextPattern;
-                        }   }   }   }
+                                U32 const newMatchIndex = matchCandidateIdx - (U32)backLength;   /* farthest position in current segment, will find a match of length currentSegmentLength + maybe some back */
+                                if (!LZ4HC_protectDictEnd(dictLimit, newMatchIndex)) {
+                                    assert(newMatchIndex >= dictLimit - 3 && newMatchIndex < dictLimit && !extDict);
+                                    matchIndex = dictLimit;
+                                } else {
+                                    matchIndex = newMatchIndex;
+                                    if (lookBackLength==0) {  /* no back possible */
+                                        size_t const maxML = MIN(currentSegmentLength, srcPatternLength);
+                                        if ((size_t)longest < maxML) {
+                                            assert(base + matchIndex < ip);
+                                            if (ip - (base+matchIndex) > LZ4_DISTANCE_MAX) break;
+                                            assert(maxML < 2 GB);
+                                            longest = (int)maxML;
+                                            *matchpos = base + matchIndex;   /* virtual pos, relative to ip, to retrieve offset */
+                                            *startpos = ip;
+                                        }
+                                        {   U32 const distToNextPattern = DELTANEXTU16(chainTable, matchIndex);
+                                            if (distToNextPattern > matchIndex) break;  /* avoid overflow */
+                                            matchIndex -= distToNextPattern;
+                        }   }   }   }   }
                         continue;
                 }   }
         }   }   /* PA optimization */
-- 
cgit v0.12


From 38c3945de300851757d0dd76182ee28aaf8253a4 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Tue, 30 Jul 2019 23:40:58 -0700
Subject: [lz4hc] Only allow chain swapping forwards

When the match is very long and found quickly, we can do
matchLength * nbCompares iterations through the chain
swapping, which can really slow down compression.
---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 596888a..0608ec6 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -314,7 +314,7 @@ LZ4HC_InsertAndGetWiderMatch (
             if (matchIndex + (U32)longest <= ipIndex) {
                 U32 distanceToNextMatch = 1;
                 int pos;
-                for (pos = 0; pos <= longest - MINMATCH; ++pos) {
+                for (pos = matchChainPos; pos <= longest - MINMATCH; ++pos) {
                     U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + (U32)pos);
                     if (candidateDist > distanceToNextMatch) {
                         distanceToNextMatch = candidateDist;
-- 
cgit v0.12


From 064adb2e8d95698168d436afc223e8ba44e56831 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 31 Jul 2019 00:57:16 -0700
Subject: [lz4hc] Chain swap with acceleration

---
 lib/lz4hc.c | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 0608ec6..5922ed7 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -312,20 +312,26 @@ LZ4HC_InsertAndGetWiderMatch (
         if (chainSwap && matchLength==longest) {    /* better match => select a better chain */
             assert(lookBackLength==0);   /* search forward only */
             if (matchIndex + (U32)longest <= ipIndex) {
+                int const kTrigger = 4;
                 U32 distanceToNextMatch = 1;
+                int const end = longest - MINMATCH + 1;
+                int step = 1;
+                int accel = 1 << kTrigger;
                 int pos;
-                for (pos = matchChainPos; pos <= longest - MINMATCH; ++pos) {
+                for (pos = 0; pos < end; pos += step) {
                     U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + (U32)pos);
+                    step = (accel++ >> kTrigger);
                     if (candidateDist > distanceToNextMatch) {
                         distanceToNextMatch = candidateDist;
                         matchChainPos = (U32)pos;
-                }   }
+                        accel = 1 << kTrigger;
+                    }
+                }
                 if (distanceToNextMatch > 1) {
                     if (distanceToNextMatch > matchIndex) break;   /* avoid overflow */
                     matchIndex -= distanceToNextMatch;
                     continue;
-                }
-        }   }
+        }   }   }
 
         {   U32 const distNextMatch = DELTANEXTU16(chainTable, matchIndex);
             if (patternAnalysis && distNextMatch==1 && matchChainPos==0) {
-- 
cgit v0.12


From e18fbd51c1529db7c78616a64a97ac23aa57c2fc Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 6 Aug 2019 14:46:31 +0200
Subject: silence msan warning when offset==0

---
 lib/lz4.c        | 129 ++++++++++++++++++++++++++++---------------------------
 tests/.gitignore |   1 +
 2 files changed, 66 insertions(+), 64 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index c8f49cc..0849505 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -184,6 +184,60 @@
 
 
 /*-************************************
+*  Common Constants
+**************************************/
+#define MINMATCH 4
+
+#define WILDCOPYLENGTH 8
+#define LASTLITERALS   5   /* see ../doc/lz4_Block_format.md#parsing-restrictions */
+#define MFLIMIT       12   /* see ../doc/lz4_Block_format.md#parsing-restrictions */
+#define MATCH_SAFEGUARD_DISTANCE  ((2*WILDCOPYLENGTH) - MINMATCH)   /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */
+#define FASTLOOP_SAFE_DISTANCE 64
+static const int LZ4_minLength = (MFLIMIT+1);
+
+#define KB *(1 <<10)
+#define MB *(1 <<20)
+#define GB *(1U<<30)
+
+#define LZ4_DISTANCE_ABSOLUTE_MAX 65535
+#if (LZ4_DISTANCE_MAX > LZ4_DISTANCE_ABSOLUTE_MAX)   /* max supported by LZ4 format */
+#  error "LZ4_DISTANCE_MAX is too big : must be <= 65535"
+#endif
+
+#define ML_BITS  4
+#define ML_MASK  ((1U<=1)
+#  include 
+#else
+#  ifndef assert
+#    define assert(condition) ((void)0)
+#  endif
+#endif
+
+#define LZ4_STATIC_ASSERT(c)   { enum { LZ4_static_assert = 1/(int)(!!(c)) }; }   /* use after variable declarations */
+
+#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2)
+#  include 
+static int g_debuglog_enable = 1;
+#  define DEBUGLOG(l, ...) {                                  \
+                if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) {  \
+                    fprintf(stderr, __FILE__ ": ");           \
+                    fprintf(stderr, __VA_ARGS__);             \
+                    fprintf(stderr, " \n");                   \
+            }   }
+#else
+#  define DEBUGLOG(l, ...)      {}    /* disabled */
+#endif
+
+
+/*-************************************
 *  Types
 **************************************/
 #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
@@ -364,29 +418,35 @@ LZ4_wildCopy32(void* dstPtr, const void* srcPtr, void* dstEnd)
     do { memcpy(d,s,16); memcpy(d+16,s+16,16); d+=32; s+=32; } while (d= dstPtr + MINMATCH
+ * - there is at least 8 bytes available to write after dstEnd */
 LZ4_FORCE_O2_INLINE_GCC_PPC64LE void
 LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset)
 {
     BYTE v[8];
+
+    assert(dstEnd >= dstPtr + MINMATCH);
+    LZ4_write32(dstPtr, 0);   /* silence an msan warning when offset==0 */
+
     switch(offset) {
     case 1:
         memset(v, *srcPtr, 8);
-        goto copy_loop;
+        break;
     case 2:
         memcpy(v, srcPtr, 2);
         memcpy(&v[2], srcPtr, 2);
         memcpy(&v[4], &v[0], 4);
-        goto copy_loop;
+        break;
     case 4:
         memcpy(v, srcPtr, 4);
         memcpy(&v[4], srcPtr, 4);
-        goto copy_loop;
+        break;
     default:
         LZ4_memcpy_using_offset_base(dstPtr, srcPtr, dstEnd, offset);
         return;
     }
 
- copy_loop:
     memcpy(dstPtr, v, 8);
     dstPtr += 8;
     while (dstPtr < dstEnd) {
@@ -398,60 +458,6 @@ LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const si
 
 
 /*-************************************
-*  Common Constants
-**************************************/
-#define MINMATCH 4
-
-#define WILDCOPYLENGTH 8
-#define LASTLITERALS   5   /* see ../doc/lz4_Block_format.md#parsing-restrictions */
-#define MFLIMIT       12   /* see ../doc/lz4_Block_format.md#parsing-restrictions */
-#define MATCH_SAFEGUARD_DISTANCE  ((2*WILDCOPYLENGTH) - MINMATCH)   /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */
-#define FASTLOOP_SAFE_DISTANCE 64
-static const int LZ4_minLength = (MFLIMIT+1);
-
-#define KB *(1 <<10)
-#define MB *(1 <<20)
-#define GB *(1U<<30)
-
-#define LZ4_DISTANCE_ABSOLUTE_MAX 65535
-#if (LZ4_DISTANCE_MAX > LZ4_DISTANCE_ABSOLUTE_MAX)   /* max supported by LZ4 format */
-#  error "LZ4_DISTANCE_MAX is too big : must be <= 65535"
-#endif
-
-#define ML_BITS  4
-#define ML_MASK  ((1U<=1)
-#  include 
-#else
-#  ifndef assert
-#    define assert(condition) ((void)0)
-#  endif
-#endif
-
-#define LZ4_STATIC_ASSERT(c)   { enum { LZ4_static_assert = 1/(int)(!!(c)) }; }   /* use after variable declarations */
-
-#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2)
-#  include 
-static int g_debuglog_enable = 1;
-#  define DEBUGLOG(l, ...) {                                  \
-                if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) {  \
-                    fprintf(stderr, __FILE__ ": ");           \
-                    fprintf(stderr, __VA_ARGS__);             \
-                    fprintf(stderr, " \n");                   \
-            }   }
-#else
-#  define DEBUGLOG(l, ...)      {}    /* disabled */
-#endif
-
-
-/*-************************************
 *  Common functions
 **************************************/
 static unsigned LZ4_NbCommonBytes (reg_t val)
@@ -1946,12 +1952,6 @@ LZ4_decompress_generic(
             length = token & ML_MASK;
 
     _copy_match:
-            if (!partialDecoding) {
-                assert(oend > op);
-                assert(oend - op >= 4);
-                LZ4_write32(op, 0);   /* silence an msan warning when offset==0; costs <1%; */
-            }   /* note : when partialDecoding, there is no guarantee that at least 4 bytes remain available in output buffer */
-
             if (length == ML_MASK) {
               variable_length_error error = ok;
               length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error);
@@ -2013,6 +2013,7 @@ LZ4_decompress_generic(
             }
 
             if (unlikely(offset<8)) {
+                LZ4_write32(op, 0);   /* silence msan warning when offset==0 */
                 op[0] = match[0];
                 op[1] = match[1];
                 op[2] = match[2];
diff --git a/tests/.gitignore b/tests/.gitignore
index c4f9092..0d13df8 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -15,6 +15,7 @@ checkFrame
 # test artefacts
 tmp*
 versionsTest
+lz4_all.c
 
 # local tests
 afl
-- 
cgit v0.12


From 918269a4e395d88364a517b889473922ac5521ec Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Tue, 6 Aug 2019 15:24:51 -0400
Subject: Make Attaching an Empty Dict Behave the Same as Using it Directly

When using an empty dictionary, we bail out of loading or attaching it in
ways that leave the working context in potentially slightly different states.
In particular, in some paths, we will cause the currentOffset to be non-zero,
while in others we would allow it to remain 0.

This difference in behavior is perfectly harmless, but in some situations, it
can produce slight differences in the compressed output. For sanity's sake,
we currently try to maintain a strict correspondence between the behavior of
the dict attachment and the dict loading paths. This patch restores them to
behaving identically.

This shouldn't have any negative side-effects, as far as I can tell. When
writing the dict attachment code, I tried to preserve zeroed currentOffsets
when possible, since they benchmarked as very slightly faster. However, the
case of attaching an empty dictionary is probably rare enought that it's
acceptable to minisculely degrade performance in that corner case.
---
 lib/lz4.c | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 0849505..147a8d6 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1408,18 +1408,18 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
      * there are only valid offsets in the window, which allows an optimization
      * in LZ4_compress_fast_continue() where it uses noDictIssue even when the
      * dictionary isn't a full 64k. */
-
-    if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;
-    base = dictEnd - 64 KB - dict->currentOffset;
-    dict->dictionary = p;
-    dict->dictSize = (U32)(dictEnd - p);
     dict->currentOffset += 64 KB;
-    dict->tableType = tableType;
 
     if (dictSize < (int)HASH_UNIT) {
         return 0;
     }
 
+    if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;
+    base = dictEnd - dict->currentOffset;
+    dict->dictionary = p;
+    dict->dictSize = (U32)(dictEnd - p);
+    dict->tableType = tableType;
+
     while (p <= dictEnd-HASH_UNIT) {
         LZ4_putPosition(p, dict->hashTable, tableType, base);
         p+=3;
@@ -1435,15 +1435,16 @@ void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dict
      */
     LZ4_resetStream_fast(workingStream);
 
+    /* If the current offset is zero, we will never look in the
+     * external dictionary context, since there is no value a table
+     * entry can take that indicate a miss. In that case, we need
+     * to bump the offset to something non-zero.
+     */
+    if (workingStream->internal_donotuse.currentOffset == 0) {
+        workingStream->internal_donotuse.currentOffset = 64 KB;
+    }
+
     if (dictionaryStream != NULL && dictionaryStream->internal_donotuse.dictSize > 0) {
-        /* If the current offset is zero, we will never look in the
-         * external dictionary context, since there is no value a table
-         * entry can take that indicate a miss. In that case, we need
-         * to bump the offset to something non-zero.
-         */
-        if (workingStream->internal_donotuse.currentOffset == 0) {
-            workingStream->internal_donotuse.currentOffset = 64 KB;
-        }
         workingStream->internal_donotuse.dictCtx = &(dictionaryStream->internal_donotuse);
     } else {
         workingStream->internal_donotuse.dictCtx = NULL;
-- 
cgit v0.12


From 4f49d744e8de26b4d942c0c0e480f6a37dde41a5 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Tue, 6 Aug 2019 18:54:03 -0400
Subject: Add Attach Dict Debug Log

---
 lib/lz4.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/lz4.c b/lib/lz4.c
index 147a8d6..07739a7 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1429,6 +1429,10 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
 }
 
 void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream) {
+    DEBUGLOG(4, "LZ4_attach_dictionary (%p, %p, size %d)",
+        workingStream, dictionaryStream, dictionaryStream != NULL ?
+        dictionaryStream->internal_donotuse.dictSize : 0);
+
     /* Calling LZ4_resetStream_fast() here makes sure that changes will not be
      * erased by subsequent calls to LZ4_resetStream_fast() in case stream was
      * marked as having dirty context, e.g. requiring full reset.
-- 
cgit v0.12


From 4c580067199c4a7fdcacc6961842ee6d3832bb93 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Tue, 6 Aug 2019 19:08:41 -0400
Subject: Only Bump Offset When Attaching Non-Null Dictionary

We do want to bump, even if the dictionary is empty, but we **don't** want to
bump if the dictionary is null.
---
 lib/lz4.c | 36 +++++++++++++++++++++---------------
 1 file changed, 21 insertions(+), 15 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 07739a7..877d14e 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1429,9 +1429,12 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
 }
 
 void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream) {
-    DEBUGLOG(4, "LZ4_attach_dictionary (%p, %p, size %d)",
-        workingStream, dictionaryStream, dictionaryStream != NULL ?
-        dictionaryStream->internal_donotuse.dictSize : 0);
+    const LZ4_stream_t_internal* dictCtx = dictionaryStream == NULL ? NULL :
+        &(dictionaryStream->internal_donotuse);
+
+    DEBUGLOG(4, "LZ4_attach_dictionary (%p, %p, size %u)",
+             workingStream, dictionaryStream,
+             dictCtx != NULL ? dictCtx->dictSize : 0);
 
     /* Calling LZ4_resetStream_fast() here makes sure that changes will not be
      * erased by subsequent calls to LZ4_resetStream_fast() in case stream was
@@ -1439,20 +1442,23 @@ void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dict
      */
     LZ4_resetStream_fast(workingStream);
 
-    /* If the current offset is zero, we will never look in the
-     * external dictionary context, since there is no value a table
-     * entry can take that indicate a miss. In that case, we need
-     * to bump the offset to something non-zero.
-     */
-    if (workingStream->internal_donotuse.currentOffset == 0) {
-        workingStream->internal_donotuse.currentOffset = 64 KB;
-    }
+    if (dictCtx != NULL) {
+        /* If the current offset is zero, we will never look in the
+         * external dictionary context, since there is no value a table
+         * entry can take that indicate a miss. In that case, we need
+         * to bump the offset to something non-zero.
+         */
+        if (workingStream->internal_donotuse.currentOffset == 0) {
+            workingStream->internal_donotuse.currentOffset = 64 KB;
+        }
 
-    if (dictionaryStream != NULL && dictionaryStream->internal_donotuse.dictSize > 0) {
-        workingStream->internal_donotuse.dictCtx = &(dictionaryStream->internal_donotuse);
-    } else {
-        workingStream->internal_donotuse.dictCtx = NULL;
+        /* Don't actually attach an empty dictionary.
+         */
+        if (dictCtx->dictSize == 0) {
+            dictCtx = NULL;
+        }
     }
+    workingStream->internal_donotuse.dictCtx = dictCtx;
 }
 
 
-- 
cgit v0.12


From d7cad81093cd805110291f84d64d385557d0ffba Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 9 Aug 2019 10:32:26 -0700
Subject: [LZ4_compress_destSize] Fix off-by-one error

PR#756 fixed the data corruption bug, but didn't clear `ip`. PR#760
fixed that off-by-one error, but missed the case where `ip == filledIp`,
which is harder for the fuzzers to find (it took 20 days not 1 day).

Verified this fixed the issue reported by OSS-Fuzz.

Credit to OSS-Fuzz.
---
 lib/lz4.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 877d14e..9808d70 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1040,7 +1040,7 @@ _next_match:
                     ip -= matchCode - newMatchCode;
                     assert(newMatchCode < matchCode);
                     matchCode = newMatchCode;
-                    if (unlikely(ip < filledIp)) {
+                    if (unlikely(ip <= filledIp)) {
                         /* We have already filled up to filledIp so if ip ends up less than filledIp
                          * we have positions in the hash table beyond the current position. This is
                          * a problem if we reuse the hash table. So we have to remove these positions
-- 
cgit v0.12


From fdf2ef5809ca875c454510610764d9125ef2ebbd Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 15 Aug 2019 13:59:59 +0200
Subject: fixed test error

could trigger %0 on exceptional circumstances
due to wrong buffer size parameter.
---
 tests/frametest.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/frametest.c b/tests/frametest.c
index 91813bb..1b932e4 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -1059,7 +1059,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
                     size_t const mask = (1<