diff options
| -rw-r--r-- | .travis.yml | 3 | ||||
| -rw-r--r-- | Makefile | 5 | ||||
| -rw-r--r-- | contrib/gen_manual/gen_manual.cpp | 30 | ||||
| -rw-r--r-- | examples/blockStreaming_lineByLine.c | 2 | ||||
| -rw-r--r-- | examples/compress_functions.c | 25 | ||||
| -rw-r--r-- | examples/simple_buffer.c | 5 | ||||
| -rw-r--r-- | lib/lz4.c | 398 | ||||
| -rw-r--r-- | lib/lz4frame.c | 14 | ||||
| -rw-r--r-- | lib/lz4hc.c | 3 | ||||
| -rw-r--r-- | programs/lz4cli.c | 2 | ||||
| -rw-r--r-- | programs/util.h | 48 | ||||
| -rw-r--r-- | tests/frametest.c | 4 | ||||
| -rw-r--r-- | tests/fullbench.c | 4 |
13 files changed, 279 insertions, 264 deletions
diff --git a/.travis.yml b/.travis.yml index de6875b..5b0c6ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -129,13 +129,14 @@ matrix: - qemu-user-static - gcc-powerpc-linux-gnu - - env: Ubu=14.04 Cmd='make staticAnalyze' COMPILER=clang + - env: Ubu=14.04 Cmd='make staticAnalyze && make cppcheck' COMPILER=clang dist: trusty sudo: required addons: apt: packages: - clang + - cppcheck - env: Ubu=14.04 Cmd='make clean all CC=gcc-4.4 MOREFLAGS=-Werror && make clean && CFLAGS=-fPIC LDFLAGS="-pie -fPIE -D_FORTIFY_SOURCE=2" make -C programs' COMPILER=gcc-4.4 dist: trusty @@ -148,9 +148,14 @@ usan: clean usan32: clean CFLAGS="-m32 -O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T30s" NB_LOOPS=-i1 +.PHONY: staticAnalyze staticAnalyze: clean CFLAGS=-g scan-build --status-bugs -v $(MAKE) all +.PHONY: cppcheck +cppcheck: + cppcheck . --force --enable=warning,portability,performance,style --error-exitcode=1 > /dev/null + platformTest: clean @echo "\n ---- test lz4 with $(CC) compiler ----" @$(CC) -v diff --git a/contrib/gen_manual/gen_manual.cpp b/contrib/gen_manual/gen_manual.cpp index 65abd3a..bedef94 100644 --- a/contrib/gen_manual/gen_manual.cpp +++ b/contrib/gen_manual/gen_manual.cpp @@ -44,7 +44,7 @@ void trim(string& s, string characters) { size_t p = s.find_first_not_of(characters); s.erase(0, p); - + p = s.find_last_not_of(characters); if (string::npos != p) s.erase(p+1); @@ -67,14 +67,13 @@ vector<string> get_lines(vector<string>& input, int& linenum, string terminator) { vector<string> out; string line; - size_t epos; while ((size_t)linenum < input.size()) { line = input[linenum]; if (terminator.empty() && line.empty()) { linenum--; break; } - - epos = line.find(terminator); + + size_t const epos = line.find(terminator); if (!terminator.empty() && epos!=string::npos) { out.push_back(line); break; @@ -152,8 +151,9 @@ int main(int argc, char *argv[]) { continue; } - /* comments of type /**< and /*!< are detected and only function declaration is highlighted (bold) */ - if ((line.find("/**<")!=string::npos || line.find("/*!<")!=string::npos) && line.find("*/")!=string::npos) { + /* comments of type / * * < and / * ! < are detected, and only function declaration is highlighted (bold) */ + if ((line.find("/**<")!=string::npos || line.find("/*!<")!=string::npos) + && line.find("*/")!=string::npos) { sout << "<pre><b>"; print_line(sout, line); sout << "</b></pre><BR>" << endl; @@ -177,16 +177,19 @@ int main(int argc, char *argv[]) { comments = get_lines(input, linenum, "*/"); if (!comments.empty()) comments[0] = line.substr(spos+3); - if (!comments.empty()) comments[comments.size()-1] = comments[comments.size()-1].substr(0, comments[comments.size()-1].find("*/")); + if (!comments.empty()) + comments[comments.size()-1] = comments[comments.size()-1].substr(0, comments[comments.size()-1].find("*/")); for (l=0; l<comments.size(); l++) { - if (comments[l].find(" *")==0) comments[l] = comments[l].substr(2); - else if (comments[l].find(" *")==0) comments[l] = comments[l].substr(3); + if (comments[l].compare(0, 2, " *") == 0) + comments[l] = comments[l].substr(2); + else if (comments[l].compare(0, 3, " *") == 0) + comments[l] = comments[l].substr(3); trim(comments[l], "*-="); } while (!comments.empty() && comments[comments.size()-1].empty()) comments.pop_back(); // remove empty line at the end while (!comments.empty() && comments[0].empty()) comments.erase(comments.begin()); // remove empty line at the start - /* comments of type /*! mean: this is a function declaration; switch comments with declarations */ + /* comments of type / * ! mean: this is a function declaration; switch comments with declarations */ if (exclam == '!') { if (!comments.empty()) comments.erase(comments.begin()); /* remove first line like "LZ4_XXX() :" */ linenum++; @@ -194,7 +197,6 @@ int main(int argc, char *argv[]) { sout << "<pre><b>"; for (l=0; l<lines.size(); l++) { - // fprintf(stderr, "line[%d]=%s\n", l, lines[l].c_str()); print_line(sout, lines[l]); } sout << "</b><p>"; @@ -202,7 +204,7 @@ int main(int argc, char *argv[]) { print_line(sout, comments[l]); } sout << "</p></pre><BR>" << endl << endl; - } else if (exclam == '=') { /* comments of type /*= and /**= mean: use a <H3> header and show also all functions until first empty line */ + } else if (exclam == '=') { /* comments of type / * = and / * * = mean: use a <H3> header and show also all functions until first empty line */ trim(comments[0], " "); sout << "<h3>" << comments[0] << "</h3><pre>"; for (l=1; l<comments.size(); l++) { @@ -214,7 +216,7 @@ int main(int argc, char *argv[]) { print_line(sout, lines[l]); } sout << "</pre></b><BR>" << endl; - } else { /* comments of type /** and /*- mean: this is a comment; use a <H2> header for the first line */ + } else { /* comments of type / * * and / * - mean: this is a comment; use a <H2> header for the first line */ if (comments.empty()) continue; trim(comments[0], " "); @@ -244,4 +246,4 @@ int main(int argc, char *argv[]) { ostream << "</html>" << endl << "</body>" << endl; return 0; -}
\ No newline at end of file +} diff --git a/examples/blockStreaming_lineByLine.c b/examples/blockStreaming_lineByLine.c index 677c426..19c3345 100644 --- a/examples/blockStreaming_lineByLine.c +++ b/examples/blockStreaming_lineByLine.c @@ -99,7 +99,7 @@ static void test_decompress( uint16_t cmpBytes = 0; if (read_uint16(inpFp, &cmpBytes) != 1) break; - if (cmpBytes <= 0) break; + if (cmpBytes == 0) break; if (read_bin(inpFp, cmpBuf, cmpBytes) != cmpBytes) break; { diff --git a/examples/compress_functions.c b/examples/compress_functions.c index d0dca13..41d3c8c 100644 --- a/examples/compress_functions.c +++ b/examples/compress_functions.c @@ -88,7 +88,6 @@ void run_screaming(const char *message, const int code) { printf("%s\n", message); exit(code); - return; } @@ -343,19 +342,19 @@ int main(int argc, char **argv) { printf("%s", separator); printf(header_format, "Source", "Function Benchmarked", "Total Seconds", "Iterations/sec", "ns/Iteration", "% of default"); printf("%s", separator); - printf(format, "Normal Text", "LZ4_compress_default()", (double)time_taken__default / BILLION, (int)(iterations / ((double)time_taken__default /BILLION)), time_taken__default / iterations, (double)time_taken__default * 100 / time_taken__default); - printf(format, "Normal Text", "LZ4_compress_fast()", (double)time_taken__fast / BILLION, (int)(iterations / ((double)time_taken__fast /BILLION)), time_taken__fast / iterations, (double)time_taken__fast * 100 / time_taken__default); - printf(format, "Normal Text", "LZ4_compress_fast_extState()", (double)time_taken__fast_extstate / BILLION, (int)(iterations / ((double)time_taken__fast_extstate /BILLION)), time_taken__fast_extstate / iterations, (double)time_taken__fast_extstate * 100 / time_taken__default); - //printf(format, "Normal Text", "LZ4_compress_generic()", (double)time_taken__generic / BILLION, (int)(iterations / ((double)time_taken__generic /BILLION)), time_taken__generic / iterations, (double)time_taken__generic * 100 / time_taken__default); - printf(format, "Normal Text", "LZ4_decompress_safe()", (double)time_taken__decomp_safe / BILLION, (int)(iterations / ((double)time_taken__decomp_safe /BILLION)), time_taken__decomp_safe / iterations, (double)time_taken__decomp_safe * 100 / time_taken__default); - printf(format, "Normal Text", "LZ4_decompress_fast()", (double)time_taken__decomp_fast / BILLION, (int)(iterations / ((double)time_taken__decomp_fast /BILLION)), time_taken__decomp_fast / iterations, (double)time_taken__decomp_fast * 100 / time_taken__default); + printf(format, "Normal Text", "LZ4_compress_default()", (double)time_taken__default / BILLION, (int)(iterations / ((double)time_taken__default /BILLION)), (int)time_taken__default / iterations, (double)time_taken__default * 100 / time_taken__default); + printf(format, "Normal Text", "LZ4_compress_fast()", (double)time_taken__fast / BILLION, (int)(iterations / ((double)time_taken__fast /BILLION)), (int)time_taken__fast / iterations, (double)time_taken__fast * 100 / time_taken__default); + printf(format, "Normal Text", "LZ4_compress_fast_extState()", (double)time_taken__fast_extstate / BILLION, (int)(iterations / ((double)time_taken__fast_extstate /BILLION)), (int)time_taken__fast_extstate / iterations, (double)time_taken__fast_extstate * 100 / time_taken__default); + //printf(format, "Normal Text", "LZ4_compress_generic()", (double)time_taken__generic / BILLION, (int)(iterations / ((double)time_taken__generic /BILLION)), (int)time_taken__generic / iterations, (double)time_taken__generic * 100 / time_taken__default); + printf(format, "Normal Text", "LZ4_decompress_safe()", (double)time_taken__decomp_safe / BILLION, (int)(iterations / ((double)time_taken__decomp_safe /BILLION)), (int)time_taken__decomp_safe / iterations, (double)time_taken__decomp_safe * 100 / time_taken__default); + printf(format, "Normal Text", "LZ4_decompress_fast()", (double)time_taken__decomp_fast / BILLION, (int)(iterations / ((double)time_taken__decomp_fast /BILLION)), (int)time_taken__decomp_fast / iterations, (double)time_taken__decomp_fast * 100 / time_taken__default); printf(header_format, "", "", "", "", "", ""); - printf(format, "Compressible", "LZ4_compress_default()", (double)time_taken_hc__default / BILLION, (int)(iterations / ((double)time_taken_hc__default /BILLION)), time_taken_hc__default / iterations, (double)time_taken_hc__default * 100 / time_taken_hc__default); - printf(format, "Compressible", "LZ4_compress_fast()", (double)time_taken_hc__fast / BILLION, (int)(iterations / ((double)time_taken_hc__fast /BILLION)), time_taken_hc__fast / iterations, (double)time_taken_hc__fast * 100 / time_taken_hc__default); - printf(format, "Compressible", "LZ4_compress_fast_extState()", (double)time_taken_hc__fast_extstate / BILLION, (int)(iterations / ((double)time_taken_hc__fast_extstate /BILLION)), time_taken_hc__fast_extstate / iterations, (double)time_taken_hc__fast_extstate * 100 / time_taken_hc__default); - //printf(format, "Compressible", "LZ4_compress_generic()", (double)time_taken_hc__generic / BILLION, (int)(iterations / ((double)time_taken_hc__generic /BILLION)), time_taken_hc__generic / iterations, (double)time_taken_hc__generic * 100 / time_taken_hc__default); - printf(format, "Compressible", "LZ4_decompress_safe()", (double)time_taken_hc__decomp_safe / BILLION, (int)(iterations / ((double)time_taken_hc__decomp_safe /BILLION)), time_taken_hc__decomp_safe / iterations, (double)time_taken_hc__decomp_safe * 100 / time_taken_hc__default); - printf(format, "Compressible", "LZ4_decompress_fast()", (double)time_taken_hc__decomp_fast / BILLION, (int)(iterations / ((double)time_taken_hc__decomp_fast /BILLION)), time_taken_hc__decomp_fast / iterations, (double)time_taken_hc__decomp_fast * 100 / time_taken_hc__default); + printf(format, "Compressible", "LZ4_compress_default()", (double)time_taken_hc__default / BILLION, (int)(iterations / ((double)time_taken_hc__default /BILLION)), (int)time_taken_hc__default / iterations, (double)time_taken_hc__default * 100 / time_taken_hc__default); + printf(format, "Compressible", "LZ4_compress_fast()", (double)time_taken_hc__fast / BILLION, (int)(iterations / ((double)time_taken_hc__fast /BILLION)), (int)time_taken_hc__fast / iterations, (double)time_taken_hc__fast * 100 / time_taken_hc__default); + printf(format, "Compressible", "LZ4_compress_fast_extState()", (double)time_taken_hc__fast_extstate / BILLION, (int)(iterations / ((double)time_taken_hc__fast_extstate /BILLION)), (int)time_taken_hc__fast_extstate / iterations, (double)time_taken_hc__fast_extstate * 100 / time_taken_hc__default); + //printf(format, "Compressible", "LZ4_compress_generic()", (double)time_taken_hc__generic / BILLION, (int)(iterations / ((double)time_taken_hc__generic /BILLION)), (int)time_taken_hc__generic / iterations, (double)time_taken_hc__generic * 100 / time_taken_hc__default); + printf(format, "Compressible", "LZ4_decompress_safe()", (double)time_taken_hc__decomp_safe / BILLION, (int)(iterations / ((double)time_taken_hc__decomp_safe /BILLION)), (int)time_taken_hc__decomp_safe / iterations, (double)time_taken_hc__decomp_safe * 100 / time_taken_hc__default); + printf(format, "Compressible", "LZ4_decompress_fast()", (double)time_taken_hc__decomp_fast / BILLION, (int)(iterations / ((double)time_taken_hc__decomp_fast /BILLION)), (int)time_taken_hc__decomp_fast / iterations, (double)time_taken_hc__decomp_fast * 100 / time_taken_hc__default); printf("%s", separator); printf("\n"); printf("All done, ran %d iterations per test.\n", iterations); diff --git a/examples/simple_buffer.c b/examples/simple_buffer.c index 403d9e8..54e542a 100644 --- a/examples/simple_buffer.c +++ b/examples/simple_buffer.c @@ -16,10 +16,9 @@ /* * Easy show-error-and-bail function. */ -void run_screaming(const char *message, const int code) { - printf("%s\n", message); +void run_screaming(const char* message, const int code) { + printf("%s \n", message); exit(code); - return; } @@ -1414,234 +1414,236 @@ LZ4_decompress_generic( const size_t dictSize /* note : = 0 if noDict */ ) { - const BYTE* ip = (const BYTE*) src; - const BYTE* const iend = ip + srcSize; + if (src == NULL) return -1; - BYTE* op = (BYTE*) dst; - BYTE* const oend = op + outputSize; - BYTE* cpy; + { const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; - const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; - const unsigned inc32table[8] = {0, 1, 2, 1, 0, 4, 4, 4}; - const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3}; + BYTE* op = (BYTE*) dst; + BYTE* const oend = op + outputSize; + BYTE* cpy; - const int safeDecode = (endOnInput==endOnInputSize); - const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); + const BYTE* const dictEnd = (dictStart == NULL) ? NULL : dictStart + dictSize; + const unsigned inc32table[8] = {0, 1, 2, 1, 0, 4, 4, 4}; + const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3}; - /* Set up the "end" pointers for the shortcut. */ - const BYTE* const shortiend = iend - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/; - const BYTE* const shortoend = oend - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/; + const int safeDecode = (endOnInput==endOnInputSize); + const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); - DEBUGLOG(5, "LZ4_decompress_generic (srcSize:%i, dstSize:%i)", srcSize, outputSize); + /* Set up the "end" pointers for the shortcut. */ + const BYTE* const shortiend = iend - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/; + const BYTE* const shortoend = oend - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/; - /* Special cases */ - assert(lowPrefix <= op); - assert(src != NULL); - if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ - if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0 ? 1 : -1); - if ((endOnInput) && unlikely(srcSize==0)) return -1; + DEBUGLOG(5, "LZ4_decompress_generic (srcSize:%i, dstSize:%i)", srcSize, outputSize); - /* Main Loop : decode sequences */ - while (1) { - const BYTE* match; - size_t offset; - - unsigned const token = *ip++; - size_t length = token >> ML_BITS; /* literal length */ + /* Special cases */ + assert(lowPrefix <= op); + if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ + if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0 ? 1 : -1); + if ((endOnInput) && unlikely(srcSize==0)) return -1; - assert(!endOnInput || ip <= iend); /* ip < iend before the increment */ + /* Main Loop : decode sequences */ + while (1) { + const BYTE* match; + size_t offset; - /* A two-stage shortcut for the most common case: - * 1) If the literal length is 0..14, and there is enough space, - * enter the shortcut and copy 16 bytes on behalf of the literals - * (in the fast mode, only 8 bytes can be safely copied this way). - * 2) Further if the match length is 4..18, copy 18 bytes in a similar - * manner; but we ensure that there's enough space in the output for - * those 18 bytes earlier, upon entering the shortcut (in other words, - * there is a combined check for both stages). - */ - if ( (endOnInput ? length != RUN_MASK : length <= 8) - /* strictly "less than" on input, to re-enter the loop with at least one byte */ - && likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)) ) { - /* Copy the literals */ - memcpy(op, ip, endOnInput ? 16 : 8); - op += length; ip += length; - - /* The second stage: prepare for match copying, decode full info. - * If it doesn't work out, the info won't be wasted. */ - length = token & ML_MASK; /* match length */ - offset = LZ4_readLE16(ip); ip += 2; - match = op - offset; - assert(match <= op); /* check overflow */ - - /* Do not deal with overlapping matches. */ - if ( (length != ML_MASK) - && (offset >= 8) - && (dict==withPrefix64k || match >= lowPrefix) ) { - /* Copy the match. */ - memcpy(op + 0, match + 0, 8); - memcpy(op + 8, match + 8, 8); - memcpy(op +16, match +16, 2); - op += length + MINMATCH; - /* Both stages worked, load the next token. */ - continue; - } + unsigned const token = *ip++; + size_t length = token >> ML_BITS; /* literal length */ - /* The second stage didn't work out, but the info is ready. - * Propel it right to the point of match copying. */ - goto _copy_match; - } + assert(!endOnInput || ip <= iend); /* ip < iend before the increment */ - /* decode literal length */ - if (length == RUN_MASK) { - unsigned s; - if (unlikely(endOnInput ? ip >= iend-RUN_MASK : 0)) goto _output_error; /* overflow detection */ - do { - s = *ip++; - length += s; - } while ( likely(endOnInput ? ip<iend-RUN_MASK : 1) & (s==255) ); - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */ - if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */ - } + /* A two-stage shortcut for the most common case: + * 1) If the literal length is 0..14, and there is enough space, + * enter the shortcut and copy 16 bytes on behalf of the literals + * (in the fast mode, only 8 bytes can be safely copied this way). + * 2) Further if the match length is 4..18, copy 18 bytes in a similar + * manner; but we ensure that there's enough space in the output for + * those 18 bytes earlier, upon entering the shortcut (in other words, + * there is a combined check for both stages). + */ + if ( (endOnInput ? length != RUN_MASK : length <= 8) + /* strictly "less than" on input, to re-enter the loop with at least one byte */ + && likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)) ) { + /* Copy the literals */ + memcpy(op, ip, endOnInput ? 16 : 8); + op += length; ip += length; + + /* The second stage: prepare for match copying, decode full info. + * If it doesn't work out, the info won't be wasted. */ + length = token & ML_MASK; /* match length */ + offset = LZ4_readLE16(ip); ip += 2; + match = op - offset; + assert(match <= op); /* check overflow */ + + /* Do not deal with overlapping matches. */ + if ( (length != ML_MASK) + && (offset >= 8) + && (dict==withPrefix64k || match >= lowPrefix) ) { + /* Copy the match. */ + memcpy(op + 0, match + 0, 8); + memcpy(op + 8, match + 8, 8); + memcpy(op +16, match +16, 2); + op += length + MINMATCH; + /* Both stages worked, load the next token. */ + continue; + } - /* copy literals */ - cpy = op+length; - LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); - if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) ) - || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) - { - if (partialDecoding) { - if (cpy > oend) { cpy = oend; length = oend-op; } /* Partial decoding : stop in the middle of literal segment */ - if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ - } else { - if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ - if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ + /* The second stage didn't work out, but the info is ready. + * Propel it right to the point of match copying. */ + goto _copy_match; } - memcpy(op, ip, length); - ip += length; - op += length; - if (!partialDecoding || (cpy == oend)) { - /* Necessarily EOF, due to parsing restrictions */ - break; + + /* decode literal length */ + if (length == RUN_MASK) { + unsigned s; + if (unlikely(endOnInput ? ip >= iend-RUN_MASK : 0)) goto _output_error; /* overflow detection */ + do { + s = *ip++; + length += s; + } while ( likely(endOnInput ? ip<iend-RUN_MASK : 1) & (s==255) ); + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */ + if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */ } - } else { - LZ4_wildCopy(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */ - ip += length; op = cpy; - } + /* copy literals */ + cpy = op+length; + LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) ) + || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) + { + if (partialDecoding) { + if (cpy > oend) { cpy = oend; length = oend-op; } /* Partial decoding : stop in the middle of literal segment */ + if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ + } else { + if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ + if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ + } + memcpy(op, ip, length); + ip += length; + op += length; + if (!partialDecoding || (cpy == oend)) { + /* Necessarily EOF, due to parsing restrictions */ + break; + } - /* get offset */ - offset = LZ4_readLE16(ip); ip+=2; - match = op - offset; + } else { + LZ4_wildCopy(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */ + ip += length; op = cpy; + } - /* get matchlength */ - length = token & ML_MASK; + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + match = op - offset; -_copy_match: - if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ - if (!partialDecoding) { - assert(oend > op); - assert(oend - op >= 4); - LZ4_write32(op, 0); /* silence an msan warning when offset==0; costs <1%; */ - } /* note : when partialDecoding, there is no guarantee that at least 4 bytes remain available in output buffer */ + /* get matchlength */ + length = token & ML_MASK; + + _copy_match: + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ + if (!partialDecoding) { + assert(oend > op); + assert(oend - op >= 4); + LZ4_write32(op, 0); /* silence an msan warning when offset==0; costs <1%; */ + } /* note : when partialDecoding, there is no guarantee that at least 4 bytes remain available in output buffer */ + + if (length == ML_MASK) { + unsigned s; + do { + s = *ip++; + if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; + length += s; + } while (s==255); + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + } + length += MINMATCH; - if (length == ML_MASK) { - unsigned s; - do { - s = *ip++; - if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; - length += s; - } while (s==255); - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ - } - length += MINMATCH; + /* match starting within external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + if (unlikely(op+length > oend-LASTLITERALS)) { + if (partialDecoding) length = MIN(length, (size_t)(oend-op)); + else goto _output_error; /* doesn't respect parsing restriction */ + } - /* match starting within external dictionary */ - if ((dict==usingExtDict) && (match < lowPrefix)) { - if (unlikely(op+length > oend-LASTLITERALS)) { - if (partialDecoding) length = MIN(length, (size_t)(oend-op)); - else goto _output_error; /* doesn't respect parsing restriction */ + if (length <= (size_t)(lowPrefix-match)) { + /* match fits entirely within external dictionary : just copy */ + memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match stretches into both external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix - match); + size_t const restSize = length - copySize; + memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) *op++ = *copyFrom++; + } else { + memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; } - if (length <= (size_t)(lowPrefix-match)) { - /* match fits entirely within external dictionary : just copy */ - memmove(op, dictEnd - (lowPrefix-match), length); - op += length; - } else { - /* match stretches into both external dictionary and current block */ - size_t const copySize = (size_t)(lowPrefix - match); - size_t const restSize = length - copySize; - memcpy(op, dictEnd - copySize, copySize); - op += copySize; - if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ - BYTE* const endOfMatch = op + restSize; - const BYTE* copyFrom = lowPrefix; - while (op < endOfMatch) *op++ = *copyFrom++; + /* copy match within block */ + cpy = op + length; + + /* partialDecoding : may not respect endBlock parsing restrictions */ + assert(op<=oend); + if (partialDecoding && (cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { + size_t const mlen = MIN(length, (size_t)(oend-op)); + const BYTE* const matchEnd = match + mlen; + BYTE* const copyEnd = op + mlen; + if (matchEnd > op) { /* overlap copy */ + while (op < copyEnd) *op++ = *match++; } else { - memcpy(op, lowPrefix, restSize); - op += restSize; - } } - continue; - } + memcpy(op, match, mlen); + } + op = copyEnd; + if (op==oend) break; + continue; + } - /* copy match within block */ - cpy = op + length; - - /* partialDecoding : may not respect endBlock parsing restrictions */ - assert(op<=oend); - if (partialDecoding && (cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { - size_t const mlen = MIN(length, (size_t)(oend-op)); - const BYTE* const matchEnd = match + mlen; - BYTE* const copyEnd = op + mlen; - if (matchEnd > op) { /* overlap copy */ - while (op < copyEnd) *op++ = *match++; + if (unlikely(offset<8)) { + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += inc32table[offset]; + memcpy(op+4, match, 4); + match -= dec64table[offset]; } else { - memcpy(op, match, mlen); + memcpy(op, match, 8); + match += 8; } - op = copyEnd; - if (op==oend) break; - continue; - } - - if (unlikely(offset<8)) { - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += inc32table[offset]; - memcpy(op+4, match, 4); - match -= dec64table[offset]; - } else { - memcpy(op, match, 8); - match += 8; - } - op += 8; - - if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { - BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1); - if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ - if (op < oCopyLimit) { - LZ4_wildCopy(op, match, oCopyLimit); - match += oCopyLimit - op; - op = oCopyLimit; + op += 8; + + if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { + BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1); + if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ + if (op < oCopyLimit) { + LZ4_wildCopy(op, match, oCopyLimit); + match += oCopyLimit - op; + op = oCopyLimit; + } + while (op < cpy) *op++ = *match++; + } else { + memcpy(op, match, 8); + if (length > 16) LZ4_wildCopy(op+8, match+8, cpy); } - while (op < cpy) *op++ = *match++; - } else { - memcpy(op, match, 8); - if (length > 16) LZ4_wildCopy(op+8, match+8, cpy); + op = cpy; /* wildcopy correction */ } - op = cpy; /* wildcopy correction */ - } - /* end of decoding */ - if (endOnInput) - return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ - else - return (int) (((const char*)ip)-src); /* Nb of input bytes read */ + /* end of decoding */ + if (endOnInput) + return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ + else + return (int) (((const char*)ip)-src); /* Nb of input bytes read */ - /* Overflow error detected */ -_output_error: - return (int) (-(((const char*)ip)-src))-1; + /* Overflow error detected */ + _output_error: + return (int) (-(((const char*)ip)-src))-1; + } } diff --git a/lib/lz4frame.c b/lib/lz4frame.c index a30de48..11e0975 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -635,7 +635,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, } } cctxPtr->tmpIn = cctxPtr->tmpBuff; cctxPtr->tmpInSize = 0; - XXH32_reset(&(cctxPtr->xxh), 0); + (void)XXH32_reset(&(cctxPtr->xxh), 0); /* context init */ cctxPtr->cdict = cdict; @@ -644,7 +644,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, LZ4F_initStream(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel, LZ4F_blockLinked); } if (preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) { - LZ4_favorDecompressionSpeed((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed); + LZ4_favorDecompressionSpeed((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed); } /* Magic Number */ @@ -887,7 +887,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, } if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) - XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize); + (void)XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize); cctxPtr->totalInSize += srcSize; return dstPtr - dstStart; @@ -1366,7 +1366,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, break; case dstage_init: - if (dctx->frameInfo.contentChecksumFlag) XXH32_reset(&(dctx->xxh), 0); + if (dctx->frameInfo.contentChecksumFlag) (void)XXH32_reset(&(dctx->xxh), 0); /* internal buffers allocation */ { size_t const bufferNeeded = dctx->maxBlockSize + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB); @@ -1431,7 +1431,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, /* next block is uncompressed */ dctx->tmpInTarget = nextCBlockSize; if (dctx->frameInfo.blockChecksumFlag) { - XXH32_reset(&dctx->blockChecksum, 0); + (void)XXH32_reset(&dctx->blockChecksum, 0); } dctx->dStage = dstage_copyDirect; break; @@ -1451,10 +1451,10 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, size_t const sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize); memcpy(dstPtr, srcPtr, sizeToCopy); if (dctx->frameInfo.blockChecksumFlag) { - XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy); + (void)XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy); } if (dctx->frameInfo.contentChecksumFlag) - XXH32_update(&dctx->xxh, srcPtr, sizeToCopy); + (void)XXH32_update(&dctx->xxh, srcPtr, sizeToCopy); if (dctx->frameInfo.contentSize) dctx->frameRemainingSize -= sizeToCopy; diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 87f4cdb..714ab62 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -231,7 +231,6 @@ LZ4HC_InsertAndGetWiderMatch ( int matchChainPos = 0; U32 const pattern = LZ4_read32(ip); U32 matchIndex; - U32 dictMatchIndex; repeat_state_e repeat = rep_untested; size_t srcPatternLength = 0; @@ -349,8 +348,8 @@ LZ4HC_InsertAndGetWiderMatch ( if (dict == usingDictCtx && nbAttempts && ipIndex - lowestMatchIndex < MAX_DISTANCE) { size_t const dictEndOffset = dictCtx->end - dictCtx->base; + U32 dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; assert(dictEndOffset <= 1 GB); - dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; matchIndex = dictMatchIndex + lowestMatchIndex - (U32)dictEndOffset; while (ipIndex - matchIndex <= MAX_DISTANCE && nbAttempts--) { const BYTE* const matchPtr = dictCtx->base + dictMatchIndex; diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 26a8089..46a95cd 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -141,7 +141,7 @@ static int usage_advanced(const char* exeName) DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n"); DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); DISPLAY( "--favor-decSpeed: compressed files decompress faster, but are less compressed \n"); - DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %u)\n", 1); + DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %i)\n", 1); DISPLAY( "Benchmark arguments : \n"); DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n"); DISPLAY( " -e# : test all compression levels from -bX to # (default : 1)\n"); diff --git a/programs/util.h b/programs/util.h index d74db0d..4aba6a3 100644 --- a/programs/util.h +++ b/programs/util.h @@ -34,6 +34,7 @@ extern "C" { #include <stdlib.h> /* malloc */ #include <string.h> /* strlen, strncpy */ #include <stdio.h> /* fprintf */ +#include <assert.h> #include <sys/types.h> /* stat, utime */ #include <sys/stat.h> /* stat */ #if defined(_MSC_VER) @@ -375,9 +376,9 @@ UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFi * A modified version of realloc(). * If UTIL_realloc() fails the original block is freed. */ -UTIL_STATIC void *UTIL_realloc(void *ptr, size_t size) +UTIL_STATIC void* UTIL_realloc(void* ptr, size_t size) { - void *newptr = realloc(ptr, size); + void* newptr = realloc(ptr, size); if (newptr) return newptr; free(ptr); return NULL; @@ -387,10 +388,10 @@ UTIL_STATIC void *UTIL_realloc(void *ptr, size_t size) #ifdef _WIN32 # define UTIL_HAS_CREATEFILELIST -UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) +UTIL_STATIC int UTIL_prepareFileList(const char* dirName, char** bufStart, size_t* pos, char** bufEnd) { char* path; - int dirLength, fnameLength, pathLength, nbFiles = 0; + int dirLength, nbFiles = 0; WIN32_FIND_DATAA cFile; HANDLE hFile; @@ -411,7 +412,8 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ free(path); do { - fnameLength = (int)strlen(cFile.cFileName); + int pathLength; + int const fnameLength = (int)strlen(cFile.cFileName); path = (char*) malloc(dirLength + fnameLength + 2); if (!path) { FindClose(hFile); return 0; } memcpy(path, dirName, dirLength); @@ -451,12 +453,11 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ # include <dirent.h> /* opendir, readdir */ # include <string.h> /* strerror, memcpy */ -UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) +UTIL_STATIC int UTIL_prepareFileList(const char* dirName, char** bufStart, size_t* pos, char** bufEnd) { - DIR *dir; - struct dirent *entry; - char* path; - int dirLength, fnameLength, pathLength, nbFiles = 0; + DIR* dir; + struct dirent * entry; + int dirLength, nbFiles = 0; if (!(dir = opendir(dirName))) { fprintf(stderr, "Cannot open directory '%s': %s\n", dirName, strerror(errno)); @@ -466,6 +467,8 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ dirLength = (int)strlen(dirName); errno = 0; while ((entry = readdir(dir)) != NULL) { + char* path; + int fnameLength, pathLength; if (strcmp (entry->d_name, "..") == 0 || strcmp (entry->d_name, ".") == 0) continue; fnameLength = (int)strlen(entry->d_name); @@ -508,7 +511,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ #else -UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) +UTIL_STATIC int UTIL_prepareFileList(const char* dirName, char** bufStart, size_t* pos, char** bufEnd) { (void)bufStart; (void)bufEnd; (void)pos; fprintf(stderr, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName); @@ -523,12 +526,13 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ * After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer) * In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called. */ -UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned inputNamesNb, char** allocatedBuffer, unsigned* allocatedNamesNb) +UTIL_STATIC const char** +UTIL_createFileList(const char** inputNames, unsigned inputNamesNb, char** allocatedBuffer, unsigned* allocatedNamesNb) { size_t pos; unsigned i, nbFiles; char* buf = (char*)malloc(LIST_SIZE_INCREASE); - char* bufend = buf + LIST_SIZE_INCREASE; + size_t bufSize = LIST_SIZE_INCREASE; const char** fileTable; if (!buf) return NULL; @@ -536,20 +540,23 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i for (i=0, pos=0, nbFiles=0; i<inputNamesNb; i++) { if (!UTIL_isDirectory(inputNames[i])) { size_t const len = strlen(inputNames[i]); - if (buf + pos + len >= bufend) { - ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE; + if (pos + len >= bufSize) { + size_t newListSize = bufSize + LIST_SIZE_INCREASE; buf = (char*)UTIL_realloc(buf, newListSize); - bufend = buf + newListSize; + bufSize = newListSize; if (!buf) return NULL; } - if (buf + pos + len < bufend) { - strncpy(buf + pos, inputNames[i], bufend - (buf + pos)); + if (pos + len < bufSize) { + strncpy(buf + pos, inputNames[i], bufSize - pos); pos += len + 1; nbFiles++; } } else { + char* bufend = buf + bufSize; nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend); if (buf == NULL) return NULL; + assert(bufend > buf); + bufSize = bufend - buf; } } if (nbFiles == 0) { free(buf); return NULL; } @@ -562,7 +569,7 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i pos += strlen(fileTable[i]) + 1; } - if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; } + if (pos > bufSize) { free(buf); free((void*)fileTable); return NULL; } /* can this happen ? */ *allocatedBuffer = buf; *allocatedNamesNb = nbFiles; @@ -571,7 +578,8 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i } -UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer) +UTIL_STATIC void +UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer) { if (allocatedBuffer) free(allocatedBuffer); if (filenameTable) free((void*)filenameTable); diff --git a/tests/frametest.c b/tests/frametest.c index 4efeb6f..5714732 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -520,7 +520,7 @@ int basicTests(U32 seed, double compressibility) LZ4F_CDict* const cdict = LZ4F_createCDict(CNBuffer, dictSize); if (cdict == NULL) goto _output_error; CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) ); - + DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with NULL dict : "); CHECK_V(cSizeNoDict, LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity, @@ -840,7 +840,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi size_t const iSize = MIN(sampleMax, (size_t)(iend-ip)); size_t const oSize = LZ4F_compressBound(iSize, prefsPtr); cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1); - DISPLAYLEVEL(6, "Sending %zi bytes to compress (stableSrc:%u) \n", + DISPLAYLEVEL(6, "Sending %zu bytes to compress (stableSrc:%u) \n", iSize, cOptions.stableSrc); result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions); diff --git a/tests/fullbench.c b/tests/fullbench.c index fd1202d..4b4e921 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -367,7 +367,6 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) size_t readSize; int compressedBuffSize; U32 crcOriginal; - size_t errorCode; /* Check file existence */ if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } @@ -561,7 +560,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) case 8: decompressionFunction = local_LZ4_decompress_safe_forceExtDict; dName = "LZ4_decompress_safe_forceExtDict"; break; #endif case 9: decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress"; - errorCode = LZ4F_compressFrame(compressed_buff, compressedBuffSize, orig_buff, benchedSize, NULL); + { size_t const errorCode = LZ4F_compressFrame(compressed_buff, compressedBuffSize, orig_buff, benchedSize, NULL); if (LZ4F_isError(errorCode)) { DISPLAY("Error while preparing compressed frame\n"); free(orig_buff); @@ -573,6 +572,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) chunkP[0].compressedSize = (int)errorCode; nbChunks = 1; break; + } default : continue; /* skip if unknown ID */ } |