diff options
| -rw-r--r-- | .travis.yml | 3 | ||||
| -rw-r--r-- | Makefile | 19 | ||||
| -rw-r--r-- | NEWS | 1 | ||||
| -rw-r--r-- | contrib/gen_manual/gen_manual.cpp | 30 | ||||
| -rw-r--r-- | doc/images/usingCDict_1_8_2.png | bin | 81858 -> 0 bytes | |||
| -rw-r--r-- | doc/lz4_manual.html | 168 | ||||
| -rw-r--r-- | doc/lz4frame_manual.html | 39 | ||||
| -rw-r--r-- | examples/blockStreaming_lineByLine.c | 2 | ||||
| -rw-r--r-- | examples/compress_functions.c | 25 | ||||
| -rw-r--r-- | examples/frameCompress.c | 21 | ||||
| -rw-r--r-- | examples/simple_buffer.c | 5 | ||||
| -rw-r--r-- | lib/Makefile | 2 | ||||
| -rw-r--r-- | lib/README.md | 17 | ||||
| -rw-r--r-- | lib/lz4.c | 488 | ||||
| -rw-r--r-- | lib/lz4.h | 221 | ||||
| -rw-r--r-- | lib/lz4frame.c | 61 | ||||
| -rw-r--r-- | lib/lz4frame.h | 43 | ||||
| -rw-r--r-- | lib/lz4hc.c | 67 | ||||
| -rw-r--r-- | lib/lz4hc.h | 88 | ||||
| -rw-r--r-- | lib/xxhash.c | 614 | ||||
| -rw-r--r-- | lib/xxhash.h | 221 | ||||
| -rw-r--r-- | programs/lz4cli.c | 16 | ||||
| -rw-r--r-- | programs/lz4io.c | 66 | ||||
| -rw-r--r-- | programs/lz4io.h | 4 | ||||
| -rw-r--r-- | programs/util.h | 48 | ||||
| -rw-r--r-- | tests/Makefile | 16 | ||||
| -rw-r--r-- | tests/checkFrame.c | 306 | ||||
| -rw-r--r-- | tests/frametest.c | 27 | ||||
| -rw-r--r-- | tests/fullbench.c | 4 | ||||
| -rw-r--r-- | tests/fuzzer.c | 43 | ||||
| -rwxr-xr-x | tests/test_custom_block_sizes.sh | 72 |
31 files changed, 1751 insertions, 986 deletions
diff --git a/.travis.yml b/.travis.yml index de6875b..5b0c6ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -129,13 +129,14 @@ matrix: - qemu-user-static - gcc-powerpc-linux-gnu - - env: Ubu=14.04 Cmd='make staticAnalyze' COMPILER=clang + - env: Ubu=14.04 Cmd='make staticAnalyze && make cppcheck' COMPILER=clang dist: trusty sudo: required addons: apt: packages: - clang + - cppcheck - env: Ubu=14.04 Cmd='make clean all CC=gcc-4.4 MOREFLAGS=-Werror && make clean && CFLAGS=-fPIC LDFLAGS="-pie -fPIE -D_FORTIFY_SOURCE=2" make -C programs' COMPILER=gcc-4.4 dist: trusty @@ -50,10 +50,10 @@ endif default: lib-release lz4-release .PHONY: all -all: allmost manuals +all: allmost examples manuals .PHONY: allmost -allmost: lib lz4 examples +allmost: lib lz4 .PHONY: lib lib-release liblz4.a lib: liblz4.a @@ -148,9 +148,14 @@ usan: clean usan32: clean CFLAGS="-m32 -O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T30s" NB_LOOPS=-i1 +.PHONY: staticAnalyze staticAnalyze: clean CFLAGS=-g scan-build --status-bugs -v $(MAKE) all +.PHONY: cppcheck +cppcheck: + cppcheck . --force --enable=warning,portability,performance,style --error-exitcode=1 > /dev/null + platformTest: clean @echo "\n ---- test lz4 with $(CC) compiler ----" @$(CC) -v @@ -181,10 +186,10 @@ ctocpptest: clean CC=$(TESTCC) $(MAKE) -C $(TESTDIR) CFLAGS="$(CFLAGS)" all c_standards: clean - CFLAGS="-std=c90 -Werror" $(MAKE) clean allmost - CFLAGS="-std=gnu90 -Werror" $(MAKE) clean allmost - CFLAGS="-std=c99 -Werror" $(MAKE) clean allmost - CFLAGS="-std=gnu99 -Werror" $(MAKE) clean allmost - CFLAGS="-std=c11 -Werror" $(MAKE) clean allmost + $(MAKE) clean; CFLAGS="-std=c90 -Werror -pedantic -Wno-long-long -Wno-variadic-macros" $(MAKE) allmost + $(MAKE) clean; CFLAGS="-std=gnu90 -Werror -pedantic -Wno-long-long -Wno-variadic-macros" $(MAKE) allmost + $(MAKE) clean; CFLAGS="-std=c99 -Werror -pedantic" $(MAKE) all + $(MAKE) clean; CFLAGS="-std=gnu99 -Werror -pedantic" $(MAKE) all + $(MAKE) clean; CFLAGS="-std=c11 -Werror" $(MAKE) all endif @@ -2,6 +2,7 @@ v1.8.3 perf: minor decompression speed improvement (~+2%) with gcc fix : corruption in v1.8.2 at level 9 for files > 64KB under rare conditions (#560) cli : new command --fast, by @jennifermliu +cli : fixed elapsed time, and added cpu load indicator (on -vv) (#555) api : LZ4_decompress_safe_partial() now decodes exactly the nb of bytes requested (feature request #566) build : added Haiku target, by @fbrosson, and MidnightBSD, by @laffer1 doc : updated documentation regarding dictionary compression diff --git a/contrib/gen_manual/gen_manual.cpp b/contrib/gen_manual/gen_manual.cpp index 65abd3a..bedef94 100644 --- a/contrib/gen_manual/gen_manual.cpp +++ b/contrib/gen_manual/gen_manual.cpp @@ -44,7 +44,7 @@ void trim(string& s, string characters) { size_t p = s.find_first_not_of(characters); s.erase(0, p); - + p = s.find_last_not_of(characters); if (string::npos != p) s.erase(p+1); @@ -67,14 +67,13 @@ vector<string> get_lines(vector<string>& input, int& linenum, string terminator) { vector<string> out; string line; - size_t epos; while ((size_t)linenum < input.size()) { line = input[linenum]; if (terminator.empty() && line.empty()) { linenum--; break; } - - epos = line.find(terminator); + + size_t const epos = line.find(terminator); if (!terminator.empty() && epos!=string::npos) { out.push_back(line); break; @@ -152,8 +151,9 @@ int main(int argc, char *argv[]) { continue; } - /* comments of type /**< and /*!< are detected and only function declaration is highlighted (bold) */ - if ((line.find("/**<")!=string::npos || line.find("/*!<")!=string::npos) && line.find("*/")!=string::npos) { + /* comments of type / * * < and / * ! < are detected, and only function declaration is highlighted (bold) */ + if ((line.find("/**<")!=string::npos || line.find("/*!<")!=string::npos) + && line.find("*/")!=string::npos) { sout << "<pre><b>"; print_line(sout, line); sout << "</b></pre><BR>" << endl; @@ -177,16 +177,19 @@ int main(int argc, char *argv[]) { comments = get_lines(input, linenum, "*/"); if (!comments.empty()) comments[0] = line.substr(spos+3); - if (!comments.empty()) comments[comments.size()-1] = comments[comments.size()-1].substr(0, comments[comments.size()-1].find("*/")); + if (!comments.empty()) + comments[comments.size()-1] = comments[comments.size()-1].substr(0, comments[comments.size()-1].find("*/")); for (l=0; l<comments.size(); l++) { - if (comments[l].find(" *")==0) comments[l] = comments[l].substr(2); - else if (comments[l].find(" *")==0) comments[l] = comments[l].substr(3); + if (comments[l].compare(0, 2, " *") == 0) + comments[l] = comments[l].substr(2); + else if (comments[l].compare(0, 3, " *") == 0) + comments[l] = comments[l].substr(3); trim(comments[l], "*-="); } while (!comments.empty() && comments[comments.size()-1].empty()) comments.pop_back(); // remove empty line at the end while (!comments.empty() && comments[0].empty()) comments.erase(comments.begin()); // remove empty line at the start - /* comments of type /*! mean: this is a function declaration; switch comments with declarations */ + /* comments of type / * ! mean: this is a function declaration; switch comments with declarations */ if (exclam == '!') { if (!comments.empty()) comments.erase(comments.begin()); /* remove first line like "LZ4_XXX() :" */ linenum++; @@ -194,7 +197,6 @@ int main(int argc, char *argv[]) { sout << "<pre><b>"; for (l=0; l<lines.size(); l++) { - // fprintf(stderr, "line[%d]=%s\n", l, lines[l].c_str()); print_line(sout, lines[l]); } sout << "</b><p>"; @@ -202,7 +204,7 @@ int main(int argc, char *argv[]) { print_line(sout, comments[l]); } sout << "</p></pre><BR>" << endl << endl; - } else if (exclam == '=') { /* comments of type /*= and /**= mean: use a <H3> header and show also all functions until first empty line */ + } else if (exclam == '=') { /* comments of type / * = and / * * = mean: use a <H3> header and show also all functions until first empty line */ trim(comments[0], " "); sout << "<h3>" << comments[0] << "</h3><pre>"; for (l=1; l<comments.size(); l++) { @@ -214,7 +216,7 @@ int main(int argc, char *argv[]) { print_line(sout, lines[l]); } sout << "</pre></b><BR>" << endl; - } else { /* comments of type /** and /*- mean: this is a comment; use a <H2> header for the first line */ + } else { /* comments of type / * * and / * - mean: this is a comment; use a <H2> header for the first line */ if (comments.empty()) continue; trim(comments[0], " "); @@ -244,4 +246,4 @@ int main(int argc, char *argv[]) { ostream << "</html>" << endl << "</body>" << endl; return 0; -}
\ No newline at end of file +} diff --git a/doc/images/usingCDict_1_8_2.png b/doc/images/usingCDict_1_8_2.png Binary files differdeleted file mode 100644 index 9434198..0000000 --- a/doc/images/usingCDict_1_8_2.png +++ /dev/null diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html index 6ebf8d2..4c4734a 100644 --- a/doc/lz4_manual.html +++ b/doc/lz4_manual.html @@ -16,12 +16,12 @@ <li><a href="#Chapter6">Streaming Compression Functions</a></li> <li><a href="#Chapter7">Streaming Decompression Functions</a></li> <li><a href="#Chapter8">Unstable declarations</a></li> -<li><a href="#Chapter9">Private definitions</a></li> +<li><a href="#Chapter9">PRIVATE DEFINITIONS</a></li> <li><a href="#Chapter10">Obsolete Functions</a></li> </ol> <hr> <a name="Chapter1"></a><h2>Introduction</h2><pre> - LZ4 is lossless compression algorithm, providing compression speed at 400 MB/s per core, + LZ4 is lossless compression algorithm, providing compression speed at 500 MB/s per core, scalable with multi-cores CPU. It features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. @@ -37,15 +37,15 @@ An additional format, called LZ4 frame specification (doc/lz4_Frame_format.md), take care of encoding standard metadata alongside LZ4-compressed blocks. - If your application requires interoperability, it's recommended to use it. - A library is provided to take care of it, see lz4frame.h. + Frame format is required for interoperability. + It is delivered through a companion API, declared in lz4frame.h. <BR></pre> <a name="Chapter2"></a><h2>Version</h2><pre></pre> <pre><b>int LZ4_versionNumber (void); </b>/**< library version number; useful to check dll version */<b> </b></pre><BR> -<pre><b>const char* LZ4_versionString (void); </b>/**< library version string; unseful to check dll version */<b> +<pre><b>const char* LZ4_versionString (void); </b>/**< library version string; useful to check dll version */<b> </b></pre><BR> <a name="Chapter3"></a><h2>Tuning parameter</h2><pre></pre> @@ -53,8 +53,8 @@ # define LZ4_MEMORY_USAGE 14 #endif </b><p> Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) - Increasing memory usage improves compression ratio - Reduced memory usage may improve speed, thanks to cache effect + Increasing memory usage improves compression ratio. + Reduced memory usage may improve speed, thanks to better cache locality. Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache </p></pre><BR> @@ -68,21 +68,21 @@ It also runs faster, so it's a recommended setting. If the function cannot compress 'src' into a more limited 'dst' budget, compression stops *immediately*, and the function result is zero. - Note : as a consequence, 'dst' content is not valid. - Note 2 : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). + In which case, 'dst' content is undefined (invalid). srcSize : max supported value is LZ4_MAX_INPUT_SIZE. dstCapacity : size of buffer 'dst' (which must be already allocated) - return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) - or 0 if compression fails + @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) + or 0 if compression fails + Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). </p></pre><BR> <pre><b>int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity); </b><p> compressedSize : is the exact complete size of the compressed block. dstCapacity : is the size of destination buffer, which must be already allocated. - return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) + @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) If destination buffer is not large enough, decoding will stop and output an error code (negative value). If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function is protected against malicious data packets. + Note : This function is protected against malicious data packets (never writes outside 'dst' buffer, nor read outside 'source' buffer). </p></pre><BR> <a name="Chapter5"></a><h2>Advanced Functions</h2><pre></pre> @@ -107,10 +107,11 @@ <pre><b>int LZ4_sizeofState(void); int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); -</b><p> Same compression function, just using an externally allocated memory space to store compression state. - Use LZ4_sizeofState() to know how much memory must be allocated, - and allocate it on 8-bytes boundaries (using malloc() typically). - Then, provide this buffer as 'void* state' to compression function. +</b><p> Same as LZ4_compress_fast(), using an externally allocated memory space for its state. + Use LZ4_sizeofState() to know how much memory must be allocated, + and allocate it on 8-bytes boundaries (using `malloc()` typically). + Then, provide this buffer as `void* state` to compression function. + </p></pre><BR> <pre><b>int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize); @@ -132,7 +133,7 @@ int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int src and now `LZ4_decompress_safe()` can be as fast and sometimes faster than `LZ4_decompress_fast()`. Moreover, LZ4_decompress_fast() is not protected vs malformed input, as it doesn't perform full validation of compressed data. As a consequence, this function is no longer recommended, and may be deprecated in future versions. - It's only remaining specificity is that it can decompress data without knowing its compressed size. + It's last remaining specificity is that it can decompress data without knowing its compressed size. originalSize : is the uncompressed size to regenerate. `dst` must be already allocated, its size must be >= 'originalSize' bytes. @@ -175,13 +176,6 @@ int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int src <a name="Chapter6"></a><h2>Streaming Compression Functions</h2><pre></pre> -<pre><b>LZ4_stream_t* LZ4_createStream(void); -int LZ4_freeStream (LZ4_stream_t* streamPtr); -</b><p> LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure. - LZ4_freeStream() releases its memory. - -</p></pre><BR> - <pre><b>void LZ4_resetStream (LZ4_stream_t* streamPtr); </b><p> An LZ4_stream_t structure can be allocated once and re-used multiple times. Use this function to start compressing a new stream. @@ -198,7 +192,7 @@ int LZ4_freeStream (LZ4_stream_t* streamPtr); <pre><b>int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); </b><p> Compress 'src' content using data from previously compressed blocks, for better compression ratio. - 'dst' buffer must be already allocated. + 'dst' buffer must be already allocated. If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. @return : size of compressed block @@ -206,10 +200,10 @@ int LZ4_freeStream (LZ4_stream_t* streamPtr); Note 1 : Each invocation to LZ4_compress_fast_continue() generates a new block. Each block has precise boundaries. + Each block must be decompressed separately, calling LZ4_decompress_*() with relevant metadata. It's not possible to append blocks together and expect a single invocation of LZ4_decompress_*() to decompress them together. - Each block must be decompressed separately, calling LZ4_decompress_*() with associated metadata. - Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory! + Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory ! Note 3 : When input is structured as a double-buffer, each buffer can have any size, including < 64 KB. Make sure that buffers are separated, by at least one byte. @@ -217,7 +211,7 @@ int LZ4_freeStream (LZ4_stream_t* streamPtr); Note 4 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. - Note 5 : After an error, the stream status is invalid, it can only be reset or freed. + Note 5 : After an error, the stream status is undefined (invalid), it can only be reset or freed. </p></pre><BR> @@ -250,7 +244,7 @@ int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); </p></pre><BR> <pre><b>int LZ4_decoderRingBufferSize(int maxBlockSize); -#define LZ4_DECODER_RING_BUFFER_SIZE(mbs) (65536 + 14 + (mbs)) </b>/* for static allocation; mbs presumed valid */<b> +#define LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize) (65536 + 14 + (maxBlockSize)) </b>/* for static allocation; maxBlockSize presumed valid */<b> </b><p> Note : in a ring buffer scenario (optional), blocks are presumed decompressed next to each other up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize), @@ -295,32 +289,37 @@ int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, </b><p> These decoding functions work the same as a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() They are stand-alone, and don't need an LZ4_streamDecode_t structure. - Dictionary is presumed stable : it must remain accessible and unmodified during next decompression. + Dictionary is presumed stable : it must remain accessible and unmodified during decompression. + Performance tip : Decompression speed can be substantially increased + when dst == dictStart + dictSize. </p></pre><BR> <a name="Chapter8"></a><h2>Unstable declarations</h2><pre> - Declarations in this section should be considered unstable. - Use at your own peril, etc., etc. - They may be removed in the future. - Their signatures may change. + Declarations in this section must be considered unstable. + Their signatures may change, or may be removed in the future. + They are therefore only safe to depend on + when the caller is statically linked against the library. + To access their declarations, define LZ4_STATIC_LINKING_ONLY. <BR></pre> -<pre><b>void LZ4_resetStream_fast (LZ4_stream_t* streamPtr); -</b><p> Use this, like LZ4_resetStream(), to prepare a context for a new chain of - calls to a streaming API (e.g., LZ4_compress_fast_continue()). +<pre><b>LZ4LIB_STATIC_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr); +</b><p> Use this to prepare a context for a new chain of calls to a streaming API + (e.g., LZ4_compress_fast_continue()). Note: - Using this in advance of a non- streaming-compression function is redundant, - and potentially bad for performance, since they all perform their own custom - reset internally. + To stay on the safe side, when LZ4_stream_t is used for the first time, + it should be either created using LZ4_createStream() or + initialized using LZ4_resetStream(). + + Note: + Using this in advance of a non-streaming-compression function is redundant, + since they all perform their own custom reset internally. Differences from LZ4_resetStream(): - When an LZ4_stream_t is known to be in a internally coherent state, - it can often be prepared for a new compression with almost no work, only - sometimes falling back to the full, expensive reset that is always required - when the stream is in an indeterminate state (i.e., the reset performed by - LZ4_resetStream()). + When an LZ4_stream_t is known to be in an internally coherent state, + it will be prepared for a new compression with almost no work. + Otherwise, it will fall back to the full, expensive reset. LZ4_streams are guaranteed to be in a valid state when: - returned from LZ4_createStream() @@ -333,28 +332,29 @@ int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, call that fully reset the state (e.g., LZ4_compress_fast_extState()) and that returned success - When a stream isn't known to be in a valid state, it is not safe to pass to - any fastReset or streaming function. It must first be cleansed by the full - LZ4_resetStream(). + Note: + A stream that was used in a compression call that did not return success + (e.g., LZ4_compress_fast_continue()), can still be passed to this function, + however, it's history is not preserved because of previous compression + failure. </p></pre><BR> -<pre><b>int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); +<pre><b>LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); </b><p> A variant of LZ4_compress_fast_extState(). - Using this variant avoids an expensive initialization step. It is only safe - to call if the state buffer is known to be correctly initialized already - (see above comment on LZ4_resetStream_fast() for a definition of "correctly - initialized"). From a high level, the difference is that this function - initializes the provided state with a call to something like - LZ4_resetStream_fast() while LZ4_compress_fast_extState() starts with a - call to LZ4_resetStream(). + Using this variant avoids an expensive initialization step. + It is only safe to call if the state buffer is known to be correctly initialized already + (see above comment on LZ4_resetStream_fast() for a definition of "correctly initialized"). + From a high level, the difference is that + this function initializes the provided state with a call to something like LZ4_resetStream_fast() + while LZ4_compress_fast_extState() starts with a call to LZ4_resetStream(). </p></pre><BR> -<pre><b>void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream); -</b><p> This is an experimental API that allows for the efficient use of a - static dictionary many times. +<pre><b>LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream); +</b><p> This is an experimental API that allows + efficient use of a static dictionary many times. Rather than re-loading the dictionary buffer into a working context before each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a @@ -365,8 +365,8 @@ int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, Currently, only streams which have been prepared by LZ4_loadDict() should be expected to work. - Alternatively, the provided dictionary stream pointer may be NULL, in which - case any existing dictionary stream is unset. + Alternatively, the provided dictionaryStream may be NULL, + in which case any existing dictionary stream is unset. If a dictionary is provided, it replaces any pre-existing stream history. The dictionary contents are the only history that can be referenced and @@ -380,10 +380,10 @@ int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, </p></pre><BR> -<a name="Chapter9"></a><h2>Private definitions</h2><pre> - Do not use these definitions. - They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. - Using these definitions will expose code to API and/or ABI break in future versions of the library. +<a name="Chapter9"></a><h2>PRIVATE DEFINITIONS</h2><pre> + Do not use these definitions directly. + They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. + Accessing members will expose code to API and/or ABI break in future versions of the library. <BR></pre> <pre><b>typedef struct { @@ -395,36 +395,36 @@ int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, </b></pre><BR> <pre><b>typedef struct { const unsigned char* externalDict; - size_t extDictSize; const unsigned char* prefixEnd; + size_t extDictSize; size_t prefixSize; } LZ4_streamDecode_t_internal; </b></pre><BR> -<pre><b>#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) +<pre><b>#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4 + ((sizeof(void*)==16) ? 4 : 0) </b>/*AS-400*/ )<b> #define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) union LZ4_stream_u { unsigned long long table[LZ4_STREAMSIZE_U64]; LZ4_stream_t_internal internal_donotuse; } ; </b>/* previously typedef'd to LZ4_stream_t */<b> -</b><p> information structure to track an LZ4 stream. - init this structure before first use. - note : only use in association with static linking ! - this definition is not API/ABI safe, - it may change in a future version ! +</b><p> information structure to track an LZ4 stream. + init this structure with LZ4_resetStream() before first use. + note : only use in association with static linking ! + this definition is not API/ABI safe, + it may change in a future version ! </p></pre><BR> -<pre><b>#define LZ4_STREAMDECODESIZE_U64 4 +<pre><b>#define LZ4_STREAMDECODESIZE_U64 (4 + ((sizeof(void*)==16) ? 2 : 0) </b>/*AS-400*/ )<b> #define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) union LZ4_streamDecode_u { unsigned long long table[LZ4_STREAMDECODESIZE_U64]; LZ4_streamDecode_t_internal internal_donotuse; } ; </b>/* previously typedef'd to LZ4_streamDecode_t */<b> -</b><p> information structure to track an LZ4 stream during decompression. - init this structure using LZ4_setStreamDecode (or memset()) before first use - note : only use in association with static linking ! - this definition is not API/ABI safe, - and may change in a future version ! +</b><p> information structure to track an LZ4 stream during decompression. + init this structure using LZ4_setStreamDecode() before first use. + note : only use in association with static linking ! + this definition is not API/ABI safe, + and may change in a future version ! </p></pre><BR> @@ -447,11 +447,11 @@ union LZ4_streamDecode_u { # define LZ4_DEPRECATED(message) # endif #endif </b>/* LZ4_DISABLE_DEPRECATE_WARNINGS */<b> -</b><p> Should deprecation warnings be a problem, - it is generally possible to disable them, - typically with -Wno-deprecated-declarations for gcc - or _CRT_SECURE_NO_WARNINGS in Visual. - Otherwise, it's also possible to define LZ4_DISABLE_DEPRECATE_WARNINGS +</b><p> Should deprecation warnings be a problem, + it is generally possible to disable them, + typically with -Wno-deprecated-declarations for gcc + or _CRT_SECURE_NO_WARNINGS in Visual. + Otherwise, it's also possible to define LZ4_DISABLE_DEPRECATE_WARNINGS </p></pre><BR> </html> diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html index fb8e0ce..2b16045 100644 --- a/doc/lz4frame_manual.html +++ b/doc/lz4frame_manual.html @@ -84,19 +84,21 @@ LZ4F_blockChecksum_t blockChecksumFlag; </b>/* 1: each block followed by a checksum of block's compressed data; 0: disabled (default) */<b> } LZ4F_frameInfo_t; </b><p> makes it possible to set or read frame parameters. - It's not required to set all fields, as long as the structure was initially memset() to zero. - For all fields, 0 sets it to default value + Structure must be first init to 0, using memset() or LZ4F_INIT_FRAMEINFO, + setting all parameters to default. + It's then possible to update selectively some parameters </p></pre><BR> <pre><b>typedef struct { LZ4F_frameInfo_t frameInfo; int compressionLevel; </b>/* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */<b> - unsigned autoFlush; </b>/* 1: always flush, to reduce usage of internal buffers */<b> - unsigned favorDecSpeed; </b>/* 1: parser favors decompression speed vs compression ratio. Only works for high compression modes (>= LZ4LZ4HC_CLEVEL_OPT_MIN) */ /* >= v1.8.2 */<b> + unsigned autoFlush; </b>/* 1: always flush; reduces usage of internal buffers */<b> + unsigned favorDecSpeed; </b>/* 1: parser favors decompression speed vs compression ratio. Only works for high compression modes (>= LZ4HC_CLEVEL_OPT_MIN) */ /* v1.8.2+ */<b> unsigned reserved[3]; </b>/* must be zero for forward compatibility */<b> } LZ4F_preferences_t; -</b><p> makes it possible to supply detailed compression parameters to the stream interface. - Structure is presumed initially memset() to zero, representing default settings. +</b><p> makes it possible to supply advanced compression instructions to streaming interface. + Structure must be first init to 0, using memset() or LZ4F_INIT_PREFERENCES, + setting all parameters to default. All reserved fields must be set to zero. </p></pre><BR> @@ -155,15 +157,19 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); </p></pre><BR> <pre><b>size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr); -</b><p> Provides minimum dstCapacity required to guarantee compression success - given a srcSize and preferences, covering worst case scenario. +</b><p> Provides minimum dstCapacity required to guarantee success of + LZ4F_compressUpdate(), given a srcSize and preferences, for a worst case scenario. + When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() instead. + Note that the result is only valid for a single invocation of LZ4F_compressUpdate(). + When invoking LZ4F_compressUpdate() multiple times, + if the output buffer is gradually filled up instead of emptied and re-used from its start, + one must check if there is enough remaining capacity before each invocation, using LZ4F_compressBound(). + @return is always the same for a srcSize and prefsPtr. prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario. - Estimation is valid for either LZ4F_compressUpdate(), LZ4F_flush() or LZ4F_compressEnd(), - Estimation includes the possibility that internal buffer might already be filled by up to (blockSize-1) bytes. - It also includes frame footer (ending + checksum), which would have to be generated by LZ4F_compressEnd(). - Estimation doesn't include frame header, as it was already generated by LZ4F_compressBegin(). - Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers. - When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. + tech details : + @return includes the possibility that internal buffer might already be filled by up to (blockSize-1) bytes. + It also includes frame footer (ending + checksum), since it might be generated by LZ4F_compressEnd(). + @return doesn't include frame header, as it was already generated by LZ4F_compressBegin(). </p></pre><BR> @@ -192,6 +198,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. @return : nb of bytes written into dstBuffer (can be zero, when there is no data stored within cctx) or an error code if it fails (which can be tested using LZ4F_isError()) + Note : LZ4F_flush() is guaranteed to be successful when dstCapacity >= LZ4F_compressBound(0, prefsPtr). </p></pre><BR> @@ -204,6 +211,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default. @return : nb of bytes written into dstBuffer, necessarily >= 4 (endMark), or an error code if it fails (which can be tested using LZ4F_isError()) + Note : LZ4F_compressEnd() is guaranteed to be successful when dstCapacity >= LZ4F_compressBound(0, prefsPtr). A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task. </p></pre><BR> @@ -295,7 +303,8 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx); and start a new one using same context resources. </p></pre><BR> -<pre><b>typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; +<pre><b>typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) + _LZ4F_dummy_error_enum_for_c89_never_used } LZ4F_errorCodes; </b></pre><BR> <a name="Chapter11"></a><h2>Bulk processing dictionary API</h2><pre></pre> diff --git a/examples/blockStreaming_lineByLine.c b/examples/blockStreaming_lineByLine.c index 677c426..19c3345 100644 --- a/examples/blockStreaming_lineByLine.c +++ b/examples/blockStreaming_lineByLine.c @@ -99,7 +99,7 @@ static void test_decompress( uint16_t cmpBytes = 0; if (read_uint16(inpFp, &cmpBytes) != 1) break; - if (cmpBytes <= 0) break; + if (cmpBytes == 0) break; if (read_bin(inpFp, cmpBuf, cmpBytes) != cmpBytes) break; { diff --git a/examples/compress_functions.c b/examples/compress_functions.c index d0dca13..41d3c8c 100644 --- a/examples/compress_functions.c +++ b/examples/compress_functions.c @@ -88,7 +88,6 @@ void run_screaming(const char *message, const int code) { printf("%s\n", message); exit(code); - return; } @@ -343,19 +342,19 @@ int main(int argc, char **argv) { printf("%s", separator); printf(header_format, "Source", "Function Benchmarked", "Total Seconds", "Iterations/sec", "ns/Iteration", "% of default"); printf("%s", separator); - printf(format, "Normal Text", "LZ4_compress_default()", (double)time_taken__default / BILLION, (int)(iterations / ((double)time_taken__default /BILLION)), time_taken__default / iterations, (double)time_taken__default * 100 / time_taken__default); - printf(format, "Normal Text", "LZ4_compress_fast()", (double)time_taken__fast / BILLION, (int)(iterations / ((double)time_taken__fast /BILLION)), time_taken__fast / iterations, (double)time_taken__fast * 100 / time_taken__default); - printf(format, "Normal Text", "LZ4_compress_fast_extState()", (double)time_taken__fast_extstate / BILLION, (int)(iterations / ((double)time_taken__fast_extstate /BILLION)), time_taken__fast_extstate / iterations, (double)time_taken__fast_extstate * 100 / time_taken__default); - //printf(format, "Normal Text", "LZ4_compress_generic()", (double)time_taken__generic / BILLION, (int)(iterations / ((double)time_taken__generic /BILLION)), time_taken__generic / iterations, (double)time_taken__generic * 100 / time_taken__default); - printf(format, "Normal Text", "LZ4_decompress_safe()", (double)time_taken__decomp_safe / BILLION, (int)(iterations / ((double)time_taken__decomp_safe /BILLION)), time_taken__decomp_safe / iterations, (double)time_taken__decomp_safe * 100 / time_taken__default); - printf(format, "Normal Text", "LZ4_decompress_fast()", (double)time_taken__decomp_fast / BILLION, (int)(iterations / ((double)time_taken__decomp_fast /BILLION)), time_taken__decomp_fast / iterations, (double)time_taken__decomp_fast * 100 / time_taken__default); + printf(format, "Normal Text", "LZ4_compress_default()", (double)time_taken__default / BILLION, (int)(iterations / ((double)time_taken__default /BILLION)), (int)time_taken__default / iterations, (double)time_taken__default * 100 / time_taken__default); + printf(format, "Normal Text", "LZ4_compress_fast()", (double)time_taken__fast / BILLION, (int)(iterations / ((double)time_taken__fast /BILLION)), (int)time_taken__fast / iterations, (double)time_taken__fast * 100 / time_taken__default); + printf(format, "Normal Text", "LZ4_compress_fast_extState()", (double)time_taken__fast_extstate / BILLION, (int)(iterations / ((double)time_taken__fast_extstate /BILLION)), (int)time_taken__fast_extstate / iterations, (double)time_taken__fast_extstate * 100 / time_taken__default); + //printf(format, "Normal Text", "LZ4_compress_generic()", (double)time_taken__generic / BILLION, (int)(iterations / ((double)time_taken__generic /BILLION)), (int)time_taken__generic / iterations, (double)time_taken__generic * 100 / time_taken__default); + printf(format, "Normal Text", "LZ4_decompress_safe()", (double)time_taken__decomp_safe / BILLION, (int)(iterations / ((double)time_taken__decomp_safe /BILLION)), (int)time_taken__decomp_safe / iterations, (double)time_taken__decomp_safe * 100 / time_taken__default); + printf(format, "Normal Text", "LZ4_decompress_fast()", (double)time_taken__decomp_fast / BILLION, (int)(iterations / ((double)time_taken__decomp_fast /BILLION)), (int)time_taken__decomp_fast / iterations, (double)time_taken__decomp_fast * 100 / time_taken__default); printf(header_format, "", "", "", "", "", ""); - printf(format, "Compressible", "LZ4_compress_default()", (double)time_taken_hc__default / BILLION, (int)(iterations / ((double)time_taken_hc__default /BILLION)), time_taken_hc__default / iterations, (double)time_taken_hc__default * 100 / time_taken_hc__default); - printf(format, "Compressible", "LZ4_compress_fast()", (double)time_taken_hc__fast / BILLION, (int)(iterations / ((double)time_taken_hc__fast /BILLION)), time_taken_hc__fast / iterations, (double)time_taken_hc__fast * 100 / time_taken_hc__default); - printf(format, "Compressible", "LZ4_compress_fast_extState()", (double)time_taken_hc__fast_extstate / BILLION, (int)(iterations / ((double)time_taken_hc__fast_extstate /BILLION)), time_taken_hc__fast_extstate / iterations, (double)time_taken_hc__fast_extstate * 100 / time_taken_hc__default); - //printf(format, "Compressible", "LZ4_compress_generic()", (double)time_taken_hc__generic / BILLION, (int)(iterations / ((double)time_taken_hc__generic /BILLION)), time_taken_hc__generic / iterations, (double)time_taken_hc__generic * 100 / time_taken_hc__default); - printf(format, "Compressible", "LZ4_decompress_safe()", (double)time_taken_hc__decomp_safe / BILLION, (int)(iterations / ((double)time_taken_hc__decomp_safe /BILLION)), time_taken_hc__decomp_safe / iterations, (double)time_taken_hc__decomp_safe * 100 / time_taken_hc__default); - printf(format, "Compressible", "LZ4_decompress_fast()", (double)time_taken_hc__decomp_fast / BILLION, (int)(iterations / ((double)time_taken_hc__decomp_fast /BILLION)), time_taken_hc__decomp_fast / iterations, (double)time_taken_hc__decomp_fast * 100 / time_taken_hc__default); + printf(format, "Compressible", "LZ4_compress_default()", (double)time_taken_hc__default / BILLION, (int)(iterations / ((double)time_taken_hc__default /BILLION)), (int)time_taken_hc__default / iterations, (double)time_taken_hc__default * 100 / time_taken_hc__default); + printf(format, "Compressible", "LZ4_compress_fast()", (double)time_taken_hc__fast / BILLION, (int)(iterations / ((double)time_taken_hc__fast /BILLION)), (int)time_taken_hc__fast / iterations, (double)time_taken_hc__fast * 100 / time_taken_hc__default); + printf(format, "Compressible", "LZ4_compress_fast_extState()", (double)time_taken_hc__fast_extstate / BILLION, (int)(iterations / ((double)time_taken_hc__fast_extstate /BILLION)), (int)time_taken_hc__fast_extstate / iterations, (double)time_taken_hc__fast_extstate * 100 / time_taken_hc__default); + //printf(format, "Compressible", "LZ4_compress_generic()", (double)time_taken_hc__generic / BILLION, (int)(iterations / ((double)time_taken_hc__generic /BILLION)), (int)time_taken_hc__generic / iterations, (double)time_taken_hc__generic * 100 / time_taken_hc__default); + printf(format, "Compressible", "LZ4_decompress_safe()", (double)time_taken_hc__decomp_safe / BILLION, (int)(iterations / ((double)time_taken_hc__decomp_safe /BILLION)), (int)time_taken_hc__decomp_safe / iterations, (double)time_taken_hc__decomp_safe * 100 / time_taken_hc__default); + printf(format, "Compressible", "LZ4_decompress_fast()", (double)time_taken_hc__decomp_fast / BILLION, (int)(iterations / ((double)time_taken_hc__decomp_fast /BILLION)), (int)time_taken_hc__decomp_fast / iterations, (double)time_taken_hc__decomp_fast * 100 / time_taken_hc__default); printf("%s", separator); printf("\n"); printf("All done, ran %d iterations per test.\n", iterations); diff --git a/examples/frameCompress.c b/examples/frameCompress.c index a0c5d3d..a189329 100644 --- a/examples/frameCompress.c +++ b/examples/frameCompress.c @@ -70,11 +70,12 @@ compress_file_internal(FILE* f_in, FILE* f_out, /* write frame header */ { size_t const headerSize = LZ4F_compressBegin(ctx, outBuff, outCapacity, &kPrefs); if (LZ4F_isError(headerSize)) { - printf("Failed to start compression: error %zu\n", headerSize); + printf("Failed to start compression: error %u \n", (unsigned)headerSize); return result; } count_out = headerSize; - printf("Buffer size is %zu bytes, header size %zu bytes\n", outCapacity, headerSize); + printf("Buffer size is %u bytes, header size %u bytes \n", + (unsigned)outCapacity, (unsigned)headerSize); safe_fwrite(outBuff, 1, headerSize, f_out); } @@ -89,11 +90,11 @@ compress_file_internal(FILE* f_in, FILE* f_out, inBuff, readSize, NULL); if (LZ4F_isError(compressedSize)) { - printf("Compression failed: error %zu\n", compressedSize); + printf("Compression failed: error %u \n", (unsigned)compressedSize); return result; } - printf("Writing %zu bytes\n", compressedSize); + printf("Writing %u bytes\n", (unsigned)compressedSize); safe_fwrite(outBuff, 1, compressedSize, f_out); count_out += compressedSize; } @@ -103,11 +104,11 @@ compress_file_internal(FILE* f_in, FILE* f_out, outBuff, outCapacity, NULL); if (LZ4F_isError(compressedSize)) { - printf("Failed to end compression: error %zu\n", compressedSize); + printf("Failed to end compression: error %u \n", (unsigned)compressedSize); return result; } - printf("Writing %zu bytes\n", compressedSize); + printf("Writing %u bytes \n", (unsigned)compressedSize); safe_fwrite(outBuff, 1, compressedSize, f_out); count_out += compressedSize; } @@ -184,8 +185,8 @@ decompress_file_internal(FILE* f_in, FILE* f_out, while (ret != 0) { /* Load more input */ size_t readSize = firstChunk ? filled : fread(src, 1, srcCapacity, f_in); firstChunk=0; - const void* srcPtr = src + alreadyConsumed; alreadyConsumed=0; - const void* const srcEnd = srcPtr + readSize; + const void* srcPtr = (const char*)src + alreadyConsumed; alreadyConsumed=0; + const void* const srcEnd = (const char*)srcPtr + readSize; if (readSize == 0 || ferror(f_in)) { printf("Decompress: not enough input or error reading file\n"); return 1; @@ -198,7 +199,7 @@ decompress_file_internal(FILE* f_in, FILE* f_out, while (srcPtr < srcEnd && ret != 0) { /* Any data within dst has been flushed at this stage */ size_t dstSize = dstCapacity; - size_t srcSize = srcEnd - srcPtr; + size_t srcSize = (const char*)srcEnd - (const char*)srcPtr; ret = LZ4F_decompress(dctx, dst, &dstSize, srcPtr, &srcSize, /* LZ4F_decompressOptions_t */ NULL); if (LZ4F_isError(ret)) { printf("Decompression error: %s\n", LZ4F_getErrorName(ret)); @@ -207,7 +208,7 @@ decompress_file_internal(FILE* f_in, FILE* f_out, /* Flush output */ if (dstSize != 0) safe_fwrite(dst, 1, dstSize, f_out); /* Update input */ - srcPtr += srcSize; + srcPtr = (const char*)srcPtr + srcSize; } assert(srcPtr <= srcEnd); diff --git a/examples/simple_buffer.c b/examples/simple_buffer.c index 403d9e8..54e542a 100644 --- a/examples/simple_buffer.c +++ b/examples/simple_buffer.c @@ -16,10 +16,9 @@ /* * Easy show-error-and-bail function. */ -void run_screaming(const char *message, const int code) { - printf("%s\n", message); +void run_screaming(const char* message, const int code) { + printf("%s \n", message); exit(code); - return; } diff --git a/lib/Makefile b/lib/Makefile index 88d9b4f..d7c8cb4 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -143,7 +143,7 @@ libdir ?= $(LIBDIR) INCLUDEDIR ?= $(prefix)/include includedir ?= $(INCLUDEDIR) -ifneq (,$(filter $(OS),OpenBSD FreeBSD NetBSD DragonFly)) +ifneq (,$(filter $(OS),OpenBSD FreeBSD NetBSD DragonFly MidnightBSD)) PKGCONFIGDIR ?= $(prefix)/libdata/pkgconfig else PKGCONFIGDIR ?= $(libdir)/pkgconfig diff --git a/lib/README.md b/lib/README.md index 7082fe3..8b3b424 100644 --- a/lib/README.md +++ b/lib/README.md @@ -16,10 +16,10 @@ They generate and decode data using [LZ4 block format]. For more compression ratio at the cost of compression speed, the High Compression variant called **lz4hc** is available. Add files **`lz4hc.c`** and **`lz4hc.h`**. -The variant still depends on regular `lib/lz4.*` source files. +This variant also depends on regular `lib/lz4.*` source files. -#### Frame variant, for interoperability +#### Frame support, for interoperability In order to produce compressed data compatible with `lz4` command line utility, it's necessary to encode lz4-compressed blocks using the [official interoperable frame format]. @@ -32,9 +32,14 @@ So it's necessary to include all `*.c` and `*.h` files present in `/lib`. #### Advanced / Experimental API -A complex API defined in `lz4frame_static.h` contains definitions -which are not guaranteed to remain stable in future versions. -As a consequence, it must be used with static linking ***only***. +Definitions which are not guaranteed to remain stable in future versions, +are protected behind macros, such as `LZ4_STATIC_LINKING_ONLY`. +As the name implies, these definitions can only be invoked +in the context of static linking ***only***. +Otherwise, dependent application may fail on API or ABI break in the future. +The associated symbols are also not present in dynamic library by default. +Should they be nonetheless needed, it's possible to force their publication +by using build macro `LZ4_PUBLISH_STATIC_FUNCTIONS`. #### Windows : using MinGW+MSYS to create DLL @@ -48,7 +53,7 @@ The dynamic library has to be added to linking options. It means that if a project that uses LZ4 consists of a single `test-dll.c` file it should be linked with `dll\liblz4.dll`. For example: ``` - gcc $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\liblz4.dll + $(CC) $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\liblz4.dll ``` The compiled executable will require LZ4 DLL which is available at `dll\liblz4.dll`. @@ -32,6 +32,13 @@ - LZ4 source repository : https://github.com/lz4/lz4 */ +/* + * LZ4_SRC_INCLUDED: + * Amalgamation flag, whether lz4.c is included + */ +#ifndef LZ4_SRC_INCLUDED +# define LZ4_SRC_INCLUDED 1 +#endif /*-************************************ * Tuning parameters @@ -455,7 +462,13 @@ static const U32 LZ4_skipTrigger = 6; /* Increase this value ==> compression ru /*-************************************ * Local Structures and types **************************************/ -typedef enum { notLimited = 0, limitedOutput = 1, fillOutput = 2 } limitedOutput_directive; +typedef enum { + noLimit = 0, + notLimited = 1, + limitedOutput = 2, + fillOutput = 3, + limitedDestSize = 4 +} limitedOutput_directive; typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t; /** @@ -522,13 +535,14 @@ static U32 LZ4_hash4(U32 sequence, tableType_t const tableType) static U32 LZ4_hash5(U64 sequence, tableType_t const tableType) { - static const U64 prime5bytes = 889523592379ULL; - static const U64 prime8bytes = 11400714785074694791ULL; const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; - if (LZ4_isLittleEndian()) + if (LZ4_isLittleEndian()) { + const U64 prime5bytes = 889523592379ULL; return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); - else + } else { + const U64 prime8bytes = 11400714785074694791ULL; return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); + } } LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) @@ -609,6 +623,15 @@ LZ4_FORCE_INLINE void LZ4_prepareTable( LZ4_stream_t_internal* const cctx, const int inputSize, const tableType_t tableType) { + /* If compression failed during the previous step, then the context + * is marked as dirty, therefore, it has to be fully reset. + */ + if (cctx->dirty) { + DEBUGLOG(5, "LZ4_prepareTable: Full reset for %p", cctx); + MEM_INIT(cctx, 0, sizeof(LZ4_stream_t_internal)); + return; + } + /* If the table hasn't been used, it's guaranteed to be zeroed out, and is * therefore safe to use no matter what mode we're in. Otherwise, we figure * out if it's safe to leave as is or whether it needs to be reset. @@ -659,6 +682,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( const dictIssue_directive dictIssue, const U32 acceleration) { + int result; const BYTE* ip = (const BYTE*) source; U32 const startIndex = cctx->currentOffset; @@ -693,9 +717,10 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( U32 forwardH; DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, tableType=%u", inputSize, tableType); - /* Init conditions */ - if (outputLimited == fillOutput && maxOutputSize < 1) return 0; /* Impossible to store anything */ - if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */ + /* If init conditions are not met, we don't have to mark stream + * as having dirty context, since no action was taken yet */ + if (outputLimited == fillOutput && maxOutputSize < 1) return 0; /* Impossible to store anything */ + if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */ if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */ assert(acceleration >= 1); @@ -813,7 +838,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( token = op++; if ((outputLimited == limitedOutput) && /* Check output buffer overflow */ (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) - return 0; + goto _failure; + if ((outputLimited == fillOutput) && (unlikely(op + (litLength+240)/255 /* litlen */ + litLength /* literals */ + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit))) { op--; @@ -886,7 +912,7 @@ _next_match: if ((outputLimited) && /* Check output buffer overflow */ (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) { if (outputLimited == limitedOutput) - return 0; + goto _failure; if (outputLimited == fillOutput) { /* Match description too long : reduce it */ U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 2 - 1 - LASTLITERALS) * 255; @@ -984,7 +1010,7 @@ _last_literals: lastRun -= (lastRun+240)/255; } if (outputLimited == limitedOutput) - return 0; + goto _failure; } if (lastRun >= RUN_MASK) { size_t accumulator = lastRun - RUN_MASK; @@ -1003,7 +1029,14 @@ _last_literals: *inputConsumed = (int) (((const char*)ip)-source); } DEBUGLOG(5, "LZ4_compress_generic: compressed %i bytes into %i bytes", inputSize, (int)(((char*)op) - dest)); - return (int)(((char*)op) - dest); + result = (int)(((char*)op) - dest); + assert(result > 0); + return result; + +_failure: + /* Mark stream as having dirty context, so, it has to be fully reset */ + cctx->dirty = 1; + return 0; } @@ -1094,23 +1127,25 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp } -int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize) +int LZ4_compress_default(const char* src, char* dst, int srcSize, int maxOutputSize) { - return LZ4_compress_fast(source, dest, inputSize, maxOutputSize, 1); + return LZ4_compress_fast(src, dst, srcSize, maxOutputSize, 1); } /* hidden debug function */ /* strangely enough, gcc generates faster code when this function is uncommented, even if unused */ -int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +int LZ4_compress_fast_force(const char* src, char* dst, int srcSize, int dstCapacity, int acceleration) { LZ4_stream_t ctx; LZ4_resetStream(&ctx); - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, sizeof(void*)==8 ? byU32 : byPtr, noDict, noDictIssue, acceleration); + if (srcSize < LZ4_64Klimit) { + return LZ4_compress_generic(&ctx.internal_donotuse, src, dst, srcSize, NULL, dstCapacity, limitedOutput, byU16, noDict, noDictIssue, acceleration); + } else { + tableType_t const addrMode = (sizeof(void*) > 4) ? byU32 : byPtr; + return LZ4_compress_generic(&ctx.internal_donotuse, src, dst, srcSize, NULL, dstCapacity, limitedOutput, addrMode, noDict, noDictIssue, acceleration); + } } @@ -1127,8 +1162,8 @@ static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src, if (*srcSizePtr < LZ4_64Klimit) { return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, byU16, noDict, noDictIssue, 1); } else { - tableType_t const tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32; - return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, tableType, noDict, noDictIssue, 1); + tableType_t const addrMode = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32; + return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, addrMode, noDict, noDictIssue, 1); } } } @@ -1159,7 +1194,7 @@ int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targe LZ4_stream_t* LZ4_createStream(void) { - LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); + LZ4_stream_t* const lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ DEBUGLOG(4, "LZ4_createStream %p", lz4s); if (lz4s == NULL) return NULL; @@ -1230,6 +1265,12 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) } void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream) { + /* Calling LZ4_resetStream_fast() here makes sure that changes will not be + * erased by subsequent calls to LZ4_resetStream_fast() in case stream was + * marked as having dirty context, e.g. requiring full reset. + */ + LZ4_resetStream_fast(working_stream); + if (dictionary_stream != NULL) { /* If the current offset is zero, we will never look in the * external dictionary context, since there is no value a table @@ -1273,7 +1314,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i)", inputSize); - if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ + if (streamPtr->dirty) return 0; /* Uninitialized structure detected */ LZ4_renormDictT(streamPtr, inputSize); /* avoid index overflow */ if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; @@ -1414,234 +1455,236 @@ LZ4_decompress_generic( const size_t dictSize /* note : = 0 if noDict */ ) { - const BYTE* ip = (const BYTE*) src; - const BYTE* const iend = ip + srcSize; + if (src == NULL) return -1; - BYTE* op = (BYTE*) dst; - BYTE* const oend = op + outputSize; - BYTE* cpy; + { const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; - const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; - const unsigned inc32table[8] = {0, 1, 2, 1, 0, 4, 4, 4}; - const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3}; + BYTE* op = (BYTE*) dst; + BYTE* const oend = op + outputSize; + BYTE* cpy; - const int safeDecode = (endOnInput==endOnInputSize); - const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); + const BYTE* const dictEnd = (dictStart == NULL) ? NULL : dictStart + dictSize; + const unsigned inc32table[8] = {0, 1, 2, 1, 0, 4, 4, 4}; + const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3}; - /* Set up the "end" pointers for the shortcut. */ - const BYTE* const shortiend = iend - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/; - const BYTE* const shortoend = oend - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/; + const int safeDecode = (endOnInput==endOnInputSize); + const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); - DEBUGLOG(5, "LZ4_decompress_generic (srcSize:%i, dstSize:%i)", srcSize, outputSize); + /* Set up the "end" pointers for the shortcut. */ + const BYTE* const shortiend = iend - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/; + const BYTE* const shortoend = oend - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/; - /* Special cases */ - assert(lowPrefix <= op); - assert(src != NULL); - if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ - if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0 ? 1 : -1); - if ((endOnInput) && unlikely(srcSize==0)) return -1; + DEBUGLOG(5, "LZ4_decompress_generic (srcSize:%i, dstSize:%i)", srcSize, outputSize); - /* Main Loop : decode sequences */ - while (1) { - const BYTE* match; - size_t offset; + /* Special cases */ + assert(lowPrefix <= op); + if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ + if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0 ? 1 : -1); + if ((endOnInput) && unlikely(srcSize==0)) return -1; - unsigned const token = *ip++; - size_t length = token >> ML_BITS; /* literal length */ + /* Main Loop : decode sequences */ + while (1) { + const BYTE* match; + size_t offset; - assert(!endOnInput || ip <= iend); /* ip < iend before the increment */ + unsigned const token = *ip++; + size_t length = token >> ML_BITS; /* literal length */ - /* A two-stage shortcut for the most common case: - * 1) If the literal length is 0..14, and there is enough space, - * enter the shortcut and copy 16 bytes on behalf of the literals - * (in the fast mode, only 8 bytes can be safely copied this way). - * 2) Further if the match length is 4..18, copy 18 bytes in a similar - * manner; but we ensure that there's enough space in the output for - * those 18 bytes earlier, upon entering the shortcut (in other words, - * there is a combined check for both stages). - */ - if ( (endOnInput ? length != RUN_MASK : length <= 8) - /* strictly "less than" on input, to re-enter the loop with at least one byte */ - && likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)) ) { - /* Copy the literals */ - memcpy(op, ip, endOnInput ? 16 : 8); - op += length; ip += length; - - /* The second stage: prepare for match copying, decode full info. - * If it doesn't work out, the info won't be wasted. */ - length = token & ML_MASK; /* match length */ - offset = LZ4_readLE16(ip); ip += 2; - match = op - offset; - assert(match <= op); /* check overflow */ - - /* Do not deal with overlapping matches. */ - if ( (length != ML_MASK) - && (offset >= 8) - && (dict==withPrefix64k || match >= lowPrefix) ) { - /* Copy the match. */ - memcpy(op + 0, match + 0, 8); - memcpy(op + 8, match + 8, 8); - memcpy(op +16, match +16, 2); - op += length + MINMATCH; - /* Both stages worked, load the next token. */ - continue; - } - - /* The second stage didn't work out, but the info is ready. - * Propel it right to the point of match copying. */ - goto _copy_match; - } + assert(!endOnInput || ip <= iend); /* ip < iend before the increment */ - /* decode literal length */ - if (length == RUN_MASK) { - unsigned s; - if (unlikely(endOnInput ? ip >= iend-RUN_MASK : 0)) goto _output_error; /* overflow detection */ - do { - s = *ip++; - length += s; - } while ( likely(endOnInput ? ip<iend-RUN_MASK : 1) & (s==255) ); - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */ - if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */ - } + /* A two-stage shortcut for the most common case: + * 1) If the literal length is 0..14, and there is enough space, + * enter the shortcut and copy 16 bytes on behalf of the literals + * (in the fast mode, only 8 bytes can be safely copied this way). + * 2) Further if the match length is 4..18, copy 18 bytes in a similar + * manner; but we ensure that there's enough space in the output for + * those 18 bytes earlier, upon entering the shortcut (in other words, + * there is a combined check for both stages). + */ + if ( (endOnInput ? length != RUN_MASK : length <= 8) + /* strictly "less than" on input, to re-enter the loop with at least one byte */ + && likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)) ) { + /* Copy the literals */ + memcpy(op, ip, endOnInput ? 16 : 8); + op += length; ip += length; + + /* The second stage: prepare for match copying, decode full info. + * If it doesn't work out, the info won't be wasted. */ + length = token & ML_MASK; /* match length */ + offset = LZ4_readLE16(ip); ip += 2; + match = op - offset; + assert(match <= op); /* check overflow */ + + /* Do not deal with overlapping matches. */ + if ( (length != ML_MASK) + && (offset >= 8) + && (dict==withPrefix64k || match >= lowPrefix) ) { + /* Copy the match. */ + memcpy(op + 0, match + 0, 8); + memcpy(op + 8, match + 8, 8); + memcpy(op +16, match +16, 2); + op += length + MINMATCH; + /* Both stages worked, load the next token. */ + continue; + } - /* copy literals */ - cpy = op+length; - LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); - if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) ) - || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) - { - if (partialDecoding) { - if (cpy > oend) { cpy = oend; length = oend-op; } /* Partial decoding : stop in the middle of literal segment */ - if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ - } else { - if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ - if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ + /* The second stage didn't work out, but the info is ready. + * Propel it right to the point of match copying. */ + goto _copy_match; } - memcpy(op, ip, length); - ip += length; - op += length; - if (!partialDecoding || (cpy == oend)) { - /* Necessarily EOF, due to parsing restrictions */ - break; + + /* decode literal length */ + if (length == RUN_MASK) { + unsigned s; + if (unlikely(endOnInput ? ip >= iend-RUN_MASK : 0)) goto _output_error; /* overflow detection */ + do { + s = *ip++; + length += s; + } while ( likely(endOnInput ? ip<iend-RUN_MASK : 1) & (s==255) ); + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */ + if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */ } - } else { - LZ4_wildCopy(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */ - ip += length; op = cpy; - } + /* copy literals */ + cpy = op+length; + LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) ) + || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) + { + if (partialDecoding) { + if (cpy > oend) { cpy = oend; length = oend-op; } /* Partial decoding : stop in the middle of literal segment */ + if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ + } else { + if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ + if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ + } + memcpy(op, ip, length); + ip += length; + op += length; + if (!partialDecoding || (cpy == oend)) { + /* Necessarily EOF, due to parsing restrictions */ + break; + } - /* get offset */ - offset = LZ4_readLE16(ip); ip+=2; - match = op - offset; + } else { + LZ4_wildCopy(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */ + ip += length; op = cpy; + } - /* get matchlength */ - length = token & ML_MASK; + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + match = op - offset; -_copy_match: - if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ - if (!partialDecoding) { - assert(oend > op); - assert(oend - op >= 4); - LZ4_write32(op, 0); /* silence an msan warning when offset==0; costs <1%; */ - } /* note : when partialDecoding, there is no guarantee that at least 4 bytes remain available in output buffer */ + /* get matchlength */ + length = token & ML_MASK; + + _copy_match: + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ + if (!partialDecoding) { + assert(oend > op); + assert(oend - op >= 4); + LZ4_write32(op, 0); /* silence an msan warning when offset==0; costs <1%; */ + } /* note : when partialDecoding, there is no guarantee that at least 4 bytes remain available in output buffer */ + + if (length == ML_MASK) { + unsigned s; + do { + s = *ip++; + if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; + length += s; + } while (s==255); + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + } + length += MINMATCH; - if (length == ML_MASK) { - unsigned s; - do { - s = *ip++; - if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; - length += s; - } while (s==255); - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ - } - length += MINMATCH; + /* match starting within external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + if (unlikely(op+length > oend-LASTLITERALS)) { + if (partialDecoding) length = MIN(length, (size_t)(oend-op)); + else goto _output_error; /* doesn't respect parsing restriction */ + } - /* match starting within external dictionary */ - if ((dict==usingExtDict) && (match < lowPrefix)) { - if (unlikely(op+length > oend-LASTLITERALS)) { - if (partialDecoding) length = MIN(length, (size_t)(oend-op)); - else goto _output_error; /* doesn't respect parsing restriction */ + if (length <= (size_t)(lowPrefix-match)) { + /* match fits entirely within external dictionary : just copy */ + memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match stretches into both external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix - match); + size_t const restSize = length - copySize; + memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) *op++ = *copyFrom++; + } else { + memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; } - if (length <= (size_t)(lowPrefix-match)) { - /* match fits entirely within external dictionary : just copy */ - memmove(op, dictEnd - (lowPrefix-match), length); - op += length; - } else { - /* match stretches into both external dictionary and current block */ - size_t const copySize = (size_t)(lowPrefix - match); - size_t const restSize = length - copySize; - memcpy(op, dictEnd - copySize, copySize); - op += copySize; - if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ - BYTE* const endOfMatch = op + restSize; - const BYTE* copyFrom = lowPrefix; - while (op < endOfMatch) *op++ = *copyFrom++; + /* copy match within block */ + cpy = op + length; + + /* partialDecoding : may not respect endBlock parsing restrictions */ + assert(op<=oend); + if (partialDecoding && (cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { + size_t const mlen = MIN(length, (size_t)(oend-op)); + const BYTE* const matchEnd = match + mlen; + BYTE* const copyEnd = op + mlen; + if (matchEnd > op) { /* overlap copy */ + while (op < copyEnd) *op++ = *match++; } else { - memcpy(op, lowPrefix, restSize); - op += restSize; - } } - continue; - } + memcpy(op, match, mlen); + } + op = copyEnd; + if (op==oend) break; + continue; + } - /* copy match within block */ - cpy = op + length; - - /* partialDecoding : may not respect endBlock parsing restrictions */ - assert(op<=oend); - if (partialDecoding && (cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { - size_t const mlen = MIN(length, (size_t)(oend-op)); - const BYTE* const matchEnd = match + mlen; - BYTE* const copyEnd = op + mlen; - if (matchEnd > op) { /* overlap copy */ - while (op < copyEnd) *op++ = *match++; + if (unlikely(offset<8)) { + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += inc32table[offset]; + memcpy(op+4, match, 4); + match -= dec64table[offset]; } else { - memcpy(op, match, mlen); + memcpy(op, match, 8); + match += 8; } - op = copyEnd; - if (op==oend) break; - continue; - } - - if (unlikely(offset<8)) { - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += inc32table[offset]; - memcpy(op+4, match, 4); - match -= dec64table[offset]; - } else { - memcpy(op, match, 8); - match += 8; - } - op += 8; - - if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { - BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1); - if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ - if (op < oCopyLimit) { - LZ4_wildCopy(op, match, oCopyLimit); - match += oCopyLimit - op; - op = oCopyLimit; + op += 8; + + if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { + BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1); + if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ + if (op < oCopyLimit) { + LZ4_wildCopy(op, match, oCopyLimit); + match += oCopyLimit - op; + op = oCopyLimit; + } + while (op < cpy) *op++ = *match++; + } else { + memcpy(op, match, 8); + if (length > 16) LZ4_wildCopy(op+8, match+8, cpy); } - while (op < cpy) *op++ = *match++; - } else { - memcpy(op, match, 8); - if (length > 16) LZ4_wildCopy(op+8, match+8, cpy); + op = cpy; /* wildcopy correction */ } - op = cpy; /* wildcopy correction */ - } - /* end of decoding */ - if (endOnInput) - return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ - else - return (int) (((const char*)ip)-src); /* Nb of input bytes read */ + /* end of decoding */ + if (endOnInput) + return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ + else + return (int) (((const char*)ip)-src); /* Nb of input bytes read */ - /* Overflow error detected */ -_output_error: - return (int) (-(((const char*)ip)-src))-1; + /* Overflow error detected */ + _output_error: + return (int) (-(((const char*)ip)-src))-1; + } } @@ -1745,12 +1788,13 @@ int LZ4_decompress_fast_doubleDict(const char* source, char* dest, int originalS LZ4_streamDecode_t* LZ4_createStreamDecode(void) { LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOC_AND_ZERO(sizeof(LZ4_streamDecode_t)); + LZ4_STATIC_ASSERT(LZ4_STREAMDECODESIZE >= sizeof(LZ4_streamDecode_t_internal)); /* A compilation error here means LZ4_STREAMDECODESIZE is not large enough */ return lz4s; } int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) { - if (!LZ4_stream) return 0; /* support free on NULL */ + if (LZ4_stream == NULL) return 0; /* support free on NULL */ FREEMEM(LZ4_stream); return 0; } @@ -103,7 +103,7 @@ extern "C" { #define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; useful to check dll version */ -LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; unseful to check dll version */ +LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; useful to check dll version */ /*-************************************ @@ -112,14 +112,15 @@ LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; /*! * LZ4_MEMORY_USAGE : * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) - * Increasing memory usage improves compression ratio - * Reduced memory usage may improve speed, thanks to cache effect + * Increasing memory usage improves compression ratio. + * Reduced memory usage may improve speed, thanks to better cache locality. * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ #ifndef LZ4_MEMORY_USAGE # define LZ4_MEMORY_USAGE 14 #endif + /*-************************************ * Simple Functions **************************************/ @@ -130,21 +131,22 @@ LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; It also runs faster, so it's a recommended setting. If the function cannot compress 'src' into a more limited 'dst' budget, compression stops *immediately*, and the function result is zero. - Note : as a consequence, 'dst' content is not valid. - Note 2 : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). + In which case, 'dst' content is undefined (invalid). srcSize : max supported value is LZ4_MAX_INPUT_SIZE. dstCapacity : size of buffer 'dst' (which must be already allocated) - return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) - or 0 if compression fails */ + @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) + or 0 if compression fails + Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). +*/ LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity); /*! LZ4_decompress_safe() : compressedSize : is the exact complete size of the compressed block. dstCapacity : is the size of destination buffer, which must be already allocated. - return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) + @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) If destination buffer is not large enough, decoding will stop and output an error code (negative value). If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function is protected against malicious data packets. + Note : This function is protected against malicious data packets (never writes outside 'dst' buffer, nor read outside 'source' buffer). */ LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity); @@ -155,8 +157,7 @@ LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSi #define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ #define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) -/*! -LZ4_compressBound() : +/*! LZ4_compressBound() : Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) This function is primarily useful for memory allocation purposes (destination buffer size). Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). @@ -167,8 +168,7 @@ LZ4_compressBound() : */ LZ4LIB_API int LZ4_compressBound(int inputSize); -/*! -LZ4_compress_fast() : +/*! LZ4_compress_fast() : Same as LZ4_compress_default(), but allows selection of "acceleration" factor. The larger the acceleration value, the faster the algorithm, but also the lesser the compression. It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. @@ -178,13 +178,12 @@ LZ4_compress_fast() : LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); -/*! -LZ4_compress_fast_extState() : - Same compression function, just using an externally allocated memory space to store compression state. - Use LZ4_sizeofState() to know how much memory must be allocated, - and allocate it on 8-bytes boundaries (using malloc() typically). - Then, provide this buffer as 'void* state' to compression function. -*/ +/*! LZ4_compress_fast_extState() : + * Same as LZ4_compress_fast(), using an externally allocated memory space for its state. + * Use LZ4_sizeofState() to know how much memory must be allocated, + * and allocate it on 8-bytes boundaries (using `malloc()` typically). + * Then, provide this buffer as `void* state` to compression function. + */ LZ4LIB_API int LZ4_sizeofState(void); LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); @@ -210,7 +209,7 @@ LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePt * and now `LZ4_decompress_safe()` can be as fast and sometimes faster than `LZ4_decompress_fast()`. * Moreover, LZ4_decompress_fast() is not protected vs malformed input, as it doesn't perform full validation of compressed data. * As a consequence, this function is no longer recommended, and may be deprecated in future versions. - * It's only remaining specificity is that it can decompress data without knowing its compressed size. + * It's last remaining specificity is that it can decompress data without knowing its compressed size. * * originalSize : is the uncompressed size to regenerate. * `dst` must be already allocated, its size must be >= 'originalSize' bytes. @@ -257,10 +256,6 @@ LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcS ***********************************************/ typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ -/*! LZ4_createStream() and LZ4_freeStream() : - * LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure. - * LZ4_freeStream() releases its memory. - */ LZ4LIB_API LZ4_stream_t* LZ4_createStream(void); LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); @@ -280,7 +275,7 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in /*! LZ4_compress_fast_continue() : * Compress 'src' content using data from previously compressed blocks, for better compression ratio. - * 'dst' buffer must be already allocated. + * 'dst' buffer must be already allocated. * If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. * * @return : size of compressed block @@ -288,10 +283,10 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in * * Note 1 : Each invocation to LZ4_compress_fast_continue() generates a new block. * Each block has precise boundaries. + * Each block must be decompressed separately, calling LZ4_decompress_*() with relevant metadata. * It's not possible to append blocks together and expect a single invocation of LZ4_decompress_*() to decompress them together. - * Each block must be decompressed separately, calling LZ4_decompress_*() with associated metadata. * - * Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory! + * Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory ! * * Note 3 : When input is structured as a double-buffer, each buffer can have any size, including < 64 KB. * Make sure that buffers are separated, by at least one byte. @@ -299,7 +294,7 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in * * Note 4 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. * - * Note 5 : After an error, the stream status is invalid, it can only be reset or freed. + * Note 5 : After an error, the stream status is undefined (invalid), it can only be reset or freed. */ LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); @@ -335,7 +330,7 @@ LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_str */ LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); -/*! LZ4_decoderRingBufferSize() : v1.8.2 +/*! LZ4_decoderRingBufferSize() : v1.8.2+ * Note : in a ring buffer scenario (optional), * blocks are presumed decompressed next to each other * up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize), @@ -347,7 +342,7 @@ LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const * or 0 if there is an error (invalid maxBlockSize). */ LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize); -#define LZ4_DECODER_RING_BUFFER_SIZE(mbs) (65536 + 14 + (mbs)) /* for static allocation; mbs presumed valid */ +#define LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize) (65536 + 14 + (maxBlockSize)) /* for static allocation; maxBlockSize presumed valid */ /*! LZ4_decompress_*_continue() : * These decoding functions allow decompression of consecutive blocks in "streaming" mode. @@ -382,42 +377,60 @@ LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecod * These decoding functions work the same as * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() * They are stand-alone, and don't need an LZ4_streamDecode_t structure. - * Dictionary is presumed stable : it must remain accessible and unmodified during next decompression. + * Dictionary is presumed stable : it must remain accessible and unmodified during decompression. + * Performance tip : Decompression speed can be substantially increased + * when dst == dictStart + dictSize. */ LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize); LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize); -/*^********************************************** +/*^************************************* * !!!!!! STATIC LINKING ONLY !!!!!! - ***********************************************/ + ***************************************/ -/*-************************************ - * Unstable declarations - ************************************** - * Declarations in this section should be considered unstable. - * Use at your own peril, etc., etc. - * They may be removed in the future. - * Their signatures may change. - **************************************/ +/*-**************************************************************************** + * Symbols declared in this section must be considered unstable. Their + * signatures or semantics may change, or they may be removed altogether in the + * future. They are therefore only safe to depend on when the caller is + * statically linked against the library. + * + * To protect against unsafe usage, not only are the declarations guarded, the + * definitions are hidden by default when building LZ4 as a shared/dynamic + * library. + * + * In order to access these declarations, define LZ4_STATIC_LINKING_ONLY in + * your application before including LZ4's headers. + * + * In order to make their implementations accessible dynamically, you must + * define LZ4_PUBLISH_STATIC_FUNCTIONS when building the LZ4 library. + ******************************************************************************/ + +#ifdef LZ4_PUBLISH_STATIC_FUNCTIONS +#define LZ4LIB_STATIC_API LZ4LIB_API +#else +#define LZ4LIB_STATIC_API +#endif #ifdef LZ4_STATIC_LINKING_ONLY /*! LZ4_resetStream_fast() : - * Use this, like LZ4_resetStream(), to prepare a context for a new chain of - * calls to a streaming API (e.g., LZ4_compress_fast_continue()). + * Use this to prepare a context for a new chain of calls to a streaming API + * (e.g., LZ4_compress_fast_continue()). * * Note: - * Using this in advance of a non- streaming-compression function is redundant, - * and potentially bad for performance, since they all perform their own custom - * reset internally. + * To stay on the safe side, when LZ4_stream_t is used for the first time, + * it should be either created using LZ4_createStream() or + * initialized using LZ4_resetStream(). + * + * Note: + * Using this in advance of a non-streaming-compression function is redundant, + * since they all perform their own custom reset internally. * * Differences from LZ4_resetStream(): - * When an LZ4_stream_t is known to be in a internally coherent state, - * it can often be prepared for a new compression with almost no work, only - * sometimes falling back to the full, expensive reset that is always required - * when the stream is in an indeterminate state (i.e., the reset performed by - * LZ4_resetStream()). + * When an LZ4_stream_t is known to be in an internally coherent state, + * it will be prepared for a new compression with almost no work. + * Otherwise, it will fall back to the full, expensive reset. * * LZ4_streams are guaranteed to be in a valid state when: * - returned from LZ4_createStream() @@ -430,28 +443,29 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or * call that fully reset the state (e.g., LZ4_compress_fast_extState()) and * that returned success * - * When a stream isn't known to be in a valid state, it is not safe to pass to - * any fastReset or streaming function. It must first be cleansed by the full - * LZ4_resetStream(). + * Note: + * A stream that was used in a compression call that did not return success + * (e.g., LZ4_compress_fast_continue()), can still be passed to this function, + * however, it's history is not preserved because of previous compression + * failure. */ -LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr); +LZ4LIB_STATIC_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr); /*! LZ4_compress_fast_extState_fastReset() : * A variant of LZ4_compress_fast_extState(). * - * Using this variant avoids an expensive initialization step. It is only safe - * to call if the state buffer is known to be correctly initialized already - * (see above comment on LZ4_resetStream_fast() for a definition of "correctly - * initialized"). From a high level, the difference is that this function - * initializes the provided state with a call to something like - * LZ4_resetStream_fast() while LZ4_compress_fast_extState() starts with a - * call to LZ4_resetStream(). + * Using this variant avoids an expensive initialization step. + * It is only safe to call if the state buffer is known to be correctly initialized already + * (see above comment on LZ4_resetStream_fast() for a definition of "correctly initialized"). + * From a high level, the difference is that + * this function initializes the provided state with a call to something like LZ4_resetStream_fast() + * while LZ4_compress_fast_extState() starts with a call to LZ4_resetStream(). */ -LZ4LIB_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); +LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); /*! LZ4_attach_dictionary() : - * This is an experimental API that allows for the efficient use of a - * static dictionary many times. + * This is an experimental API that allows + * efficient use of a static dictionary many times. * * Rather than re-loading the dictionary buffer into a working context before * each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a @@ -462,8 +476,8 @@ LZ4LIB_API int LZ4_compress_fast_extState_fastReset (void* state, const char* sr * Currently, only streams which have been prepared by LZ4_loadDict() should * be expected to work. * - * Alternatively, the provided dictionary stream pointer may be NULL, in which - * case any existing dictionary stream is unset. + * Alternatively, the provided dictionaryStream may be NULL, + * in which case any existing dictionary stream is unset. * * If a dictionary is provided, it replaces any pre-existing stream history. * The dictionary contents are the only history that can be referenced and @@ -475,17 +489,18 @@ LZ4LIB_API int LZ4_compress_fast_extState_fastReset (void* state, const char* sr * stream (and source buffer) must remain in-place / accessible / unchanged * through the completion of the first compression call on the stream. */ -LZ4LIB_API void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream); +LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream); #endif -/*-************************************ - * Private definitions - ************************************** - * Do not use these definitions. - * They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. - * Using these definitions will expose code to API and/or ABI break in future versions of the library. - **************************************/ + +/*-************************************************************ + * PRIVATE DEFINITIONS + ************************************************************** + * Do not use these definitions directly. + * They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. + * Accessing members will expose code to API and/or ABI break in future versions of the library. + **************************************************************/ #define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) #define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) #define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ @@ -497,7 +512,7 @@ typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; struct LZ4_stream_t_internal { uint32_t hashTable[LZ4_HASH_SIZE_U32]; uint32_t currentOffset; - uint16_t initCheck; + uint16_t dirty; uint16_t tableType; const uint8_t* dictionary; const LZ4_stream_t_internal* dictCtx; @@ -517,7 +532,7 @@ typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; struct LZ4_stream_t_internal { unsigned int hashTable[LZ4_HASH_SIZE_U32]; unsigned int currentOffset; - unsigned short initCheck; + unsigned short dirty; unsigned short tableType; const unsigned char* dictionary; const LZ4_stream_t_internal* dictCtx; @@ -526,22 +541,21 @@ struct LZ4_stream_t_internal { typedef struct { const unsigned char* externalDict; - size_t extDictSize; const unsigned char* prefixEnd; + size_t extDictSize; size_t prefixSize; } LZ4_streamDecode_t_internal; #endif -/*! - * LZ4_stream_t : - * information structure to track an LZ4 stream. - * init this structure before first use. - * note : only use in association with static linking ! - * this definition is not API/ABI safe, - * it may change in a future version ! +/*! LZ4_stream_t : + * information structure to track an LZ4 stream. + * init this structure with LZ4_resetStream() before first use. + * note : only use in association with static linking ! + * this definition is not API/ABI safe, + * it may change in a future version ! */ -#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) +#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4 + ((sizeof(void*)==16) ? 4 : 0) /*AS-400*/ ) #define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) union LZ4_stream_u { unsigned long long table[LZ4_STREAMSIZE_U64]; @@ -549,15 +563,14 @@ union LZ4_stream_u { } ; /* previously typedef'd to LZ4_stream_t */ -/*! - * LZ4_streamDecode_t : - * information structure to track an LZ4 stream during decompression. - * init this structure using LZ4_setStreamDecode (or memset()) before first use - * note : only use in association with static linking ! - * this definition is not API/ABI safe, - * and may change in a future version ! +/*! LZ4_streamDecode_t : + * information structure to track an LZ4 stream during decompression. + * init this structure using LZ4_setStreamDecode() before first use. + * note : only use in association with static linking ! + * this definition is not API/ABI safe, + * and may change in a future version ! */ -#define LZ4_STREAMDECODESIZE_U64 4 +#define LZ4_STREAMDECODESIZE_U64 (4 + ((sizeof(void*)==16) ? 2 : 0) /*AS-400*/ ) #define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) union LZ4_streamDecode_u { unsigned long long table[LZ4_STREAMDECODESIZE_U64]; @@ -570,11 +583,11 @@ union LZ4_streamDecode_u { **************************************/ /*! Deprecation warnings - Should deprecation warnings be a problem, - it is generally possible to disable them, - typically with -Wno-deprecated-declarations for gcc - or _CRT_SECURE_NO_WARNINGS in Visual. - Otherwise, it's also possible to define LZ4_DISABLE_DEPRECATE_WARNINGS */ + * Should deprecation warnings be a problem, + * it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc + * or _CRT_SECURE_NO_WARNINGS in Visual. + * Otherwise, it's also possible to define LZ4_DISABLE_DEPRECATE_WARNINGS */ #ifdef LZ4_DISABLE_DEPRECATE_WARNINGS # define LZ4_DEPRECATED(message) /* disable deprecation warnings */ #else @@ -594,8 +607,8 @@ union LZ4_streamDecode_u { #endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ /* Obsolete compression functions */ -LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* source, char* dest, int sourceSize); -LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* source, char* dest, int sourceSize); +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); @@ -616,8 +629,8 @@ LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompres */ LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer); LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int LZ4_sizeofStreamState(void); -LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer); -LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer (void* state); +LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer); +LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer (void* state); /* Obsolete streaming decoding functions */ LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 08bf0fa..357f962 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -33,8 +33,8 @@ You can contact the author at : */ /* LZ4F is a stand-alone API to create LZ4-compressed Frames -* in full conformance with specification v1.5.0 -* All related operations, including memory management, are handled by the library. +* in full conformance with specification v1.6.1 . +* This library rely upon memory management capabilities. * */ @@ -63,9 +63,9 @@ You can contact the author at : * Memory routines **************************************/ #include <stdlib.h> /* malloc, calloc, free */ -#define ALLOC(s) malloc(s) -#define ALLOC_AND_ZERO(s) calloc(1,s) -#define FREEMEM free +#define ALLOC(s) malloc(s) +#define ALLOC_AND_ZERO(s) calloc(1,(s)) +#define FREEMEM(p) free(p) #include <string.h> /* memset, memcpy, memmove */ #define MEM_INIT memset @@ -635,7 +635,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, } } cctxPtr->tmpIn = cctxPtr->tmpBuff; cctxPtr->tmpInSize = 0; - XXH32_reset(&(cctxPtr->xxh), 0); + (void)XXH32_reset(&(cctxPtr->xxh), 0); /* context init */ cctxPtr->cdict = cdict; @@ -644,7 +644,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, LZ4F_initStream(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel, LZ4F_blockLinked); } if (preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) { - LZ4_favorDecompressionSpeed((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed); + LZ4_favorDecompressionSpeed((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed); } /* Magic Number */ @@ -686,7 +686,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes. * preferencesPtr can be NULL, in which case default parameters are selected. * @return : number of bytes written into dstBuffer for the header - * or an error code (can be tested using LZ4F_isError()) + * or an error code (can be tested using LZ4F_isError()) */ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, @@ -887,7 +887,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, } if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) - XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize); + (void)XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize); cctxPtr->totalInSize += srcSize; return dstPtr - dstStart; @@ -934,28 +934,35 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const /*! LZ4F_compressEnd() : - * When you want to properly finish the compressed frame, just call LZ4F_compressEnd(). - * It will flush whatever data remained within compressionContext (like LZ4_flush()) - * but also properly finalize the frame, with an endMark and a checksum. - * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) - * The function outputs an error code if it fails (can be tested using LZ4F_isError()) - * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. - * compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same. + * When you want to properly finish the compressed frame, just call LZ4F_compressEnd(). + * It will flush whatever data remained within compressionContext (like LZ4_flush()) + * but also properly finalize the frame, with an endMark and an (optional) checksum. + * LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + * @return: the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) + * or an error code if it fails (can be tested using LZ4F_isError()) + * The context can then be used again to compress a new frame, starting with LZ4F_compressBegin(). */ -size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) +size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const LZ4F_compressOptions_t* compressOptionsPtr) { BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; - size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstMaxSize, compressOptionsPtr); + size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr); if (LZ4F_isError(flushSize)) return flushSize; dstPtr += flushSize; + assert(flushSize <= dstCapacity); + dstCapacity -= flushSize; + + if (dstCapacity < 4) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); LZ4F_writeLE32(dstPtr, 0); - dstPtr+=4; /* endMark */ + dstPtr += 4; /* endMark */ if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) { U32 const xxh = XXH32_digest(&(cctxPtr->xxh)); + if (dstCapacity < 8) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); LZ4F_writeLE32(dstPtr, xxh); dstPtr+=4; /* content Checksum */ } @@ -1366,7 +1373,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, break; case dstage_init: - if (dctx->frameInfo.contentChecksumFlag) XXH32_reset(&(dctx->xxh), 0); + if (dctx->frameInfo.contentChecksumFlag) (void)XXH32_reset(&(dctx->xxh), 0); /* internal buffers allocation */ { size_t const bufferNeeded = dctx->maxBlockSize + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB); @@ -1431,7 +1438,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, /* next block is uncompressed */ dctx->tmpInTarget = nextCBlockSize; if (dctx->frameInfo.blockChecksumFlag) { - XXH32_reset(&dctx->blockChecksum, 0); + (void)XXH32_reset(&dctx->blockChecksum, 0); } dctx->dStage = dstage_copyDirect; break; @@ -1451,10 +1458,10 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, size_t const sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize); memcpy(dstPtr, srcPtr, sizeToCopy); if (dctx->frameInfo.blockChecksumFlag) { - XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy); + (void)XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy); } if (dctx->frameInfo.contentChecksumFlag) - XXH32_update(&dctx->xxh, srcPtr, sizeToCopy); + (void)XXH32_update(&dctx->xxh, srcPtr, sizeToCopy); if (dctx->frameInfo.contentSize) dctx->frameRemainingSize -= sizeToCopy; @@ -1474,7 +1481,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, } dctx->tmpInTarget -= sizeToCopy; /* need to copy more */ nextSrcSizeHint = dctx->tmpInTarget + - + dctx->frameInfo.contentChecksumFlag * 4 /* block checksum */ + + dctx->frameInfo.blockChecksumFlag * 4 /* size for block checksum */ + BHSize /* next header size */; doAnotherStage = 0; break; @@ -1525,8 +1532,10 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, dctx->tmpInSize += sizeToCopy; srcPtr += sizeToCopy; if (dctx->tmpInSize < dctx->tmpInTarget) { /* need more input */ - nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize; - doAnotherStage=0; + nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + + dctx->frameInfo.blockChecksumFlag * 4 /* size for block checksum */ + + BHSize /* next header size */; + doAnotherStage = 0; break; } selectedIn = dctx->tmpIn; diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 75f1fd9..7c7c34e 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -248,6 +248,7 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); /*---- Compression ----*/ #define LZ4F_HEADER_SIZE_MAX 19 /* LZ4 Frame header size can vary from 7 to 19 bytes */ + /*! LZ4F_compressBegin() : * will write the frame header into dstBuffer. * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. @@ -260,15 +261,19 @@ LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, const LZ4F_preferences_t* prefsPtr); /*! LZ4F_compressBound() : - * Provides minimum dstCapacity required to guarantee compression success - * given a srcSize and preferences, covering worst case scenario. + * Provides minimum dstCapacity required to guarantee success of + * LZ4F_compressUpdate(), given a srcSize and preferences, for a worst case scenario. + * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() instead. + * Note that the result is only valid for a single invocation of LZ4F_compressUpdate(). + * When invoking LZ4F_compressUpdate() multiple times, + * if the output buffer is gradually filled up instead of emptied and re-used from its start, + * one must check if there is enough remaining capacity before each invocation, using LZ4F_compressBound(). + * @return is always the same for a srcSize and prefsPtr. * prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario. - * Estimation is valid for either LZ4F_compressUpdate(), LZ4F_flush() or LZ4F_compressEnd(), - * Estimation includes the possibility that internal buffer might already be filled by up to (blockSize-1) bytes. - * It also includes frame footer (ending + checksum), which would have to be generated by LZ4F_compressEnd(). - * Estimation doesn't include frame header, as it was already generated by LZ4F_compressBegin(). - * Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers. - * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. + * tech details : + * @return includes the possibility that internal buffer might already be filled by up to (blockSize-1) bytes. + * It also includes frame footer (ending + checksum), since it might be generated by LZ4F_compressEnd(). + * @return doesn't include frame header, as it was already generated by LZ4F_compressBegin(). */ LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr); @@ -295,6 +300,7 @@ LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. * @return : nb of bytes written into dstBuffer (can be zero, when there is no data stored within cctx) * or an error code if it fails (which can be tested using LZ4F_isError()) + * Note : LZ4F_flush() is guaranteed to be successful when dstCapacity >= LZ4F_compressBound(0, prefsPtr). */ LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, @@ -307,6 +313,7 @@ LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, * `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default. * @return : nb of bytes written into dstBuffer, necessarily >= 4 (endMark), * or an error code if it fails (which can be tested using LZ4F_isError()) + * Note : LZ4F_compressEnd() is guaranteed to be successful when dstCapacity >= LZ4F_compressBound(0, prefsPtr). * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task. */ LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, @@ -427,15 +434,15 @@ LZ4FLIB_API void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx); /* always su extern "C" { #endif -/* These declarations are not stable and may change in the future. They are - * therefore only safe to depend on when the caller is statically linked - * against the library. To access their declarations, define - * LZ4F_STATIC_LINKING_ONLY. +/* These declarations are not stable and may change in the future. + * They are therefore only safe to depend on + * when the caller is statically linked against the library. + * To access their declarations, define LZ4F_STATIC_LINKING_ONLY. * - * There is a further protection mechanism where these symbols aren't published - * into shared/dynamic libraries. You can override this behavior and force - * them to be published by defining LZ4F_PUBLISH_STATIC_FUNCTIONS. Use at - * your own risk. + * By default, these symbols aren't published into shared/dynamic libraries. + * You can override this behavior and force them to be published + * by defining LZ4F_PUBLISH_STATIC_FUNCTIONS. + * Use at your own risk. */ #ifdef LZ4F_PUBLISH_STATIC_FUNCTIONS #define LZ4FLIB_STATIC_API LZ4FLIB_API @@ -471,12 +478,12 @@ extern "C" { #define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM, /* enum list is exposed, to handle specific errors */ -typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; +typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) + _LZ4F_dummy_error_enum_for_c89_never_used } LZ4F_errorCodes; LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult); - /********************************** * Bulk processing dictionary API *********************************/ diff --git a/lib/lz4hc.c b/lib/lz4hc.c index e913ee7..56c8f47 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -61,9 +61,21 @@ # pragma clang diagnostic ignored "-Wunused-function" #endif +/*=== Enums ===*/ +typedef enum { noDictCtx, usingDictCtxHc } dictCtx_directive; +#ifndef LZ4_SRC_INCLUDED +typedef enum { + noLimit = 0, + limitedOutput = 1, + limitedDestSize = 2 +} limitedOutput_directive; +#endif + + #define LZ4_COMMONDEFS_ONLY +#ifndef LZ4_SRC_INCLUDED #include "lz4.c" /* LZ4_count, constants, mem */ - +#endif /*=== Constants ===*/ #define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH) @@ -79,9 +91,6 @@ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } -/*=== Enums ===*/ -typedef enum { noDictCtx, usingDictCtx } dictCtx_directive; - /************************************** * HC Compression @@ -231,7 +240,6 @@ LZ4HC_InsertAndGetWiderMatch ( int matchChainPos = 0; U32 const pattern = LZ4_read32(ip); U32 matchIndex; - U32 dictMatchIndex; repeat_state_e repeat = rep_untested; size_t srcPatternLength = 0; @@ -347,10 +355,10 @@ LZ4HC_InsertAndGetWiderMatch ( } /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */ - if (dict == usingDictCtx && nbAttempts && ipIndex - lowestMatchIndex < MAX_DISTANCE) { + if (dict == usingDictCtxHc && nbAttempts && ipIndex - lowestMatchIndex < MAX_DISTANCE) { size_t const dictEndOffset = dictCtx->end - dictCtx->base; + U32 dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; assert(dictEndOffset <= 1 GB); - dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; matchIndex = dictMatchIndex + lowestMatchIndex - (U32)dictEndOffset; while (ipIndex - matchIndex <= MAX_DISTANCE && nbAttempts--) { const BYTE* const matchPtr = dictCtx->base + dictMatchIndex; @@ -395,14 +403,6 @@ int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index tabl return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis, 0 /*chainSwap*/, dict, favorCompressionRatio); } - - -typedef enum { - noLimit = 0, - limitedOutput = 1, - limitedDestSize = 2, -} limitedOutput_directive; - /* LZ4HC_encodeSequence() : * @return : 0 if ok, * 1 if buffer issue detected */ @@ -745,16 +745,22 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal ( cLevel = MIN(LZ4HC_CLEVEL_MAX, cLevel); { cParams_t const cParam = clTable[cLevel]; HCfavor_e const favor = ctx->favorDecSpeed ? favorDecompressionSpeed : favorCompressionRatio; - if (cParam.strat == lz4hc) - return LZ4HC_compress_hashChain(ctx, + int result; + + if (cParam.strat == lz4hc) { + result = LZ4HC_compress_hashChain(ctx, src, dst, srcSizePtr, dstCapacity, cParam.nbSearches, limit, dict); - assert(cParam.strat == lz4opt); - return LZ4HC_compress_optimal(ctx, - src, dst, srcSizePtr, dstCapacity, - cParam.nbSearches, cParam.targetLength, limit, - cLevel == LZ4HC_CLEVEL_MAX, /* ultra mode */ - dict, favor); + } else { + assert(cParam.strat == lz4opt); + result = LZ4HC_compress_optimal(ctx, + src, dst, srcSizePtr, dstCapacity, + cParam.nbSearches, cParam.targetLength, limit, + cLevel == LZ4HC_CLEVEL_MAX, /* ultra mode */ + dict, favor); + } + if (result <= 0) ctx->dirty = 1; + return result; } } @@ -795,7 +801,7 @@ static int LZ4HC_compress_generic_dictCtx ( ctx->compressionLevel = (short)cLevel; return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); } else { - return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx); + return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtxHc); } } @@ -893,16 +899,21 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) LZ4_streamHCPtr->internal_donotuse.base = NULL; LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL; LZ4_streamHCPtr->internal_donotuse.favorDecSpeed = 0; + LZ4_streamHCPtr->internal_donotuse.dirty = 0; LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); } void LZ4_resetStreamHC_fast (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) { DEBUGLOG(4, "LZ4_resetStreamHC_fast(%p, %d)", LZ4_streamHCPtr, compressionLevel); - LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base; - LZ4_streamHCPtr->internal_donotuse.base = NULL; - LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL; - LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); + if (LZ4_streamHCPtr->internal_donotuse.dirty) { + LZ4_resetStreamHC(LZ4_streamHCPtr, compressionLevel); + } else { + LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base; + LZ4_streamHCPtr->internal_donotuse.base = NULL; + LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL; + LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); + } } void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 970fa39..d3fb594 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -123,12 +123,20 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in */ -/*-************************************************************** + + +/*^********************************************** + * !!!!!! STATIC LINKING ONLY !!!!!! + ***********************************************/ + +/*-****************************************************************** * PRIVATE DEFINITIONS : - * Do not use these definitions. - * They are exposed to allow static allocation of `LZ4_streamHC_t`. - * Using these definitions makes the code vulnerable to potential API break when upgrading LZ4 - ****************************************************************/ + * Do not use these definitions directly. + * They are merely exposed to allow static allocation of `LZ4_streamHC_t`. + * Declare an `LZ4_streamHC_t` directly, rather than any type below. + * Even then, only do so in the context of static linking, as definitions may change between versions. + ********************************************************************/ + #define LZ4HC_DICTIONARY_LOGSIZE 16 #define LZ4HC_MAXD (1<<LZ4HC_DICTIONARY_LOGSIZE) #define LZ4HC_MAXD_MASK (LZ4HC_MAXD - 1) @@ -153,7 +161,9 @@ struct LZ4HC_CCtx_internal uint32_t lowLimit; /* below that point, no more dict */ uint32_t nextToUpdate; /* index from which to continue dictionary update */ short compressionLevel; - short favorDecSpeed; + int8_t favorDecSpeed; /* favor decompression speed if this flag set, + otherwise, favor compression ratio */ + int8_t dirty; /* stream has to be fully reset if this flag is set */ const LZ4HC_CCtx_internal* dictCtx; }; @@ -171,26 +181,30 @@ struct LZ4HC_CCtx_internal unsigned int lowLimit; /* below that point, no more dict */ unsigned int nextToUpdate; /* index from which to continue dictionary update */ short compressionLevel; - short favorDecSpeed; + char favorDecSpeed; /* favor decompression speed if this flag set, + otherwise, favor compression ratio */ + char dirty; /* stream has to be fully reset if this flag is set */ const LZ4HC_CCtx_internal* dictCtx; }; #endif -#define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56) /* 262200 */ + +/* do not use these definitions directly. + * allocate an LZ4_streamHC_t instead. */ +#define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56 + ((sizeof(void*)==16) ? 56 : 0) /* AS400*/ ) /* 262200 or 262256*/ #define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t)) union LZ4_streamHC_u { size_t table[LZ4_STREAMHCSIZE_SIZET]; LZ4HC_CCtx_internal internal_donotuse; -}; /* previously typedef'd to LZ4_streamHC_t */ -/* - LZ4_streamHC_t : - This structure allows static allocation of LZ4 HC streaming state. - State must be initialized using LZ4_resetStreamHC() before first use. - - Static allocation shall only be used in combination with static linking. - When invoking LZ4 from a DLL, use create/free functions instead, which are API and ABI stable. -*/ +}; /* previously typedef'd to LZ4_streamHC_t */ +/* LZ4_streamHC_t : + * This structure allows static allocation of LZ4 HC streaming state. + * State must be initialized using LZ4_resetStreamHC() before first use. + * + * Static allocation shall only be used in combination with static linking. + * When invoking LZ4 from a DLL, use create/free functions instead, which are API and ABI stable. + */ /*-************************************ @@ -258,10 +272,11 @@ extern "C" { * or 0 if compression fails. * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src` */ -int LZ4_compress_HC_destSize(void* LZ4HC_Data, - const char* src, char* dst, - int* srcSizePtr, int targetDstSize, - int compressionLevel); +LZ4LIB_STATIC_API int LZ4_compress_HC_destSize( + void* LZ4HC_Data, + const char* src, char* dst, + int* srcSizePtr, int targetDstSize, + int compressionLevel); /*! LZ4_compress_HC_continue_destSize() : v1.8.0 (experimental) * Similar as LZ4_compress_HC_continue(), @@ -272,20 +287,23 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, * or 0 if compression fails. * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src`. */ -int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr, - const char* src, char* dst, - int* srcSizePtr, int targetDstSize); +LZ4LIB_STATIC_API int LZ4_compress_HC_continue_destSize( + LZ4_streamHC_t* LZ4_streamHCPtr, + const char* src, char* dst, + int* srcSizePtr, int targetDstSize); /*! LZ4_setCompressionLevel() : v1.8.0 (experimental) * It's possible to change compression level between 2 invocations of LZ4_compress_HC_continue*() */ -void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); +LZ4LIB_STATIC_API void LZ4_setCompressionLevel( + LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); /*! LZ4_favorDecompressionSpeed() : v1.8.2 (experimental) * Parser will select decisions favoring decompression over compression ratio. * Only work at highest compression settings (level >= LZ4HC_CLEVEL_OPT_MIN) */ -void LZ4_favorDecompressionSpeed(LZ4_streamHC_t* LZ4_streamHCPtr, int favor); +LZ4LIB_STATIC_API void LZ4_favorDecompressionSpeed( + LZ4_streamHC_t* LZ4_streamHCPtr, int favor); /*! LZ4_resetStreamHC_fast() : * When an LZ4_streamHC_t is known to be in a internally coherent state, @@ -304,8 +322,14 @@ void LZ4_favorDecompressionSpeed(LZ4_streamHC_t* LZ4_streamHCPtr, int favor); * - the stream was in an indeterminate state and was used in a compression * call that fully reset the state (LZ4_compress_HC_extStateHC()) and that * returned success + * + * Note: + * A stream that was last used in a compression call that returned an error + * may be passed to this function. However, it will be fully reset, which will + * clear any existing history and settings from the context. */ -void LZ4_resetStreamHC_fast(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); +LZ4LIB_STATIC_API void LZ4_resetStreamHC_fast( + LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); /*! LZ4_compress_HC_extStateHC_fastReset() : * A variant of LZ4_compress_HC_extStateHC(). @@ -318,7 +342,11 @@ void LZ4_resetStreamHC_fast(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLeve * LZ4_resetStreamHC_fast() while LZ4_compress_HC_extStateHC() starts with a * call to LZ4_resetStreamHC(). */ -int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel); +LZ4LIB_STATIC_API int LZ4_compress_HC_extStateHC_fastReset ( + void* state, + const char* src, char* dst, + int srcSize, int dstCapacity, + int compressionLevel); /*! LZ4_attach_HC_dictionary() : * This is an experimental API that allows for the efficient use of a @@ -345,7 +373,9 @@ int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* ds * stream (and source buffer) must remain in-place / accessible / unchanged * through the lifetime of the stream session. */ -LZ4LIB_API void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC_t *dictionary_stream); +LZ4LIB_STATIC_API void LZ4_attach_HC_dictionary( + LZ4_streamHC_t *working_stream, + const LZ4_streamHC_t *dictionary_stream); #if defined (__cplusplus) } diff --git a/lib/xxhash.c b/lib/xxhash.c index 3fc97fd..ff28749 100644 --- a/lib/xxhash.c +++ b/lib/xxhash.c @@ -50,20 +50,26 @@ * Prefer these methods in priority order (0 > 1 > 2) */ #ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) \ + || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) \ + || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define XXH_FORCE_MEMORY_ACCESS 2 # elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || \ - (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) + (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) \ + || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) \ + || defined(__ARM_ARCH_7S__) )) # define XXH_FORCE_MEMORY_ACCESS 1 # endif #endif /*!XXH_ACCEPT_NULL_INPUT_POINTER : - * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. - * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. - * By default, this option is disabled. To enable it, uncomment below define : + * If input pointer is NULL, xxHash default behavior is to dereference it, triggering a segfault. + * When this macro is enabled, xxHash actively checks input for null pointer. + * It it is, result for null input pointers is the same as a null-length input. */ -/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ +#ifndef XXH_ACCEPT_NULL_INPUT_POINTER /* can be defined externally */ +# define XXH_ACCEPT_NULL_INPUT_POINTER 0 +#endif /*!XXH_FORCE_NATIVE_FORMAT : * By default, xxHash library provides endian-independent Hash values, based on little-endian convention. @@ -80,8 +86,9 @@ /*!XXH_FORCE_ALIGN_CHECK : * This is a minor performance trick, only useful with lots of very small keys. * It means : check for aligned/unaligned input. - * The check costs one initial branch per hash; set to 0 when the input data - * is guaranteed to be aligned. + * The check costs one initial branch per hash; + * set it to 0 when the input is guaranteed to be aligned, + * or when alignment doesn't matter for performance. */ #ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ # if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) @@ -104,6 +111,8 @@ static void XXH_free (void* p) { free(p); } #include <string.h> static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } +#include <assert.h> /* assert */ + #define XXH_STATIC_LINKING_ONLY #include "xxhash.h" @@ -113,40 +122,35 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp ***************************************/ #ifdef _MSC_VER /* Visual Studio */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#endif - -#ifndef XXH_FORCE_INLINE -# ifdef _MSC_VER /* Visual Studio */ -# define XXH_FORCE_INLINE static __forceinline -# else -# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -# ifdef __GNUC__ -# define XXH_FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define XXH_FORCE_INLINE static inline -# endif +# define FORCE_INLINE static __forceinline +#else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) # else -# define XXH_FORCE_INLINE static -# endif /* __STDC_VERSION__ */ -# endif /* _MSC_VER */ -#endif /* XXH_FORCE_INLINE */ +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif /* ************************************* * Basic Types ***************************************/ #ifndef MEM_MODULE -# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) # include <stdint.h> typedef uint8_t BYTE; typedef uint16_t U16; typedef uint32_t U32; - typedef int32_t S32; # else typedef unsigned char BYTE; typedef unsigned short U16; typedef unsigned int U32; - typedef signed int S32; # endif #endif @@ -213,8 +217,12 @@ typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; /* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */ #ifndef XXH_CPU_LITTLE_ENDIAN - static const int g_one = 1; -# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one)) +static int XXH_isLittleEndian(void) +{ + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} +# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() #endif @@ -223,7 +231,7 @@ typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; *****************************/ typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; -XXH_FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) { if (align==XXH_unaligned) return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); @@ -231,7 +239,7 @@ XXH_FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, X return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); } -XXH_FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) +FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) { return XXH_readLE32_align(ptr, endian, XXH_unaligned); } @@ -245,12 +253,12 @@ static U32 XXH_readBE32(const void* ptr) /* ************************************* * Macros ***************************************/ -#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ +#define XXH_STATIC_ASSERT(c) { enum { XXH_sa = 1/(int)(!!(c)) }; } /* use after variable declarations */ XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } /* ******************************************************************* -* 32-bits hash functions +* 32-bit hash functions *********************************************************************/ static const U32 PRIME32_1 = 2654435761U; static const U32 PRIME32_2 = 2246822519U; @@ -266,14 +274,89 @@ static U32 XXH32_round(U32 seed, U32 input) return seed; } -XXH_FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) +/* mix all bits */ +static U32 XXH32_avalanche(U32 h32) +{ + h32 ^= h32 >> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + return(h32); +} + +#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) + +static U32 +XXH32_finalize(U32 h32, const void* ptr, size_t len, + XXH_endianess endian, XXH_alignment align) + +{ + const BYTE* p = (const BYTE*)ptr; + +#define PROCESS1 \ + h32 += (*p++) * PRIME32_5; \ + h32 = XXH_rotl32(h32, 11) * PRIME32_1 ; + +#define PROCESS4 \ + h32 += XXH_get32bits(p) * PRIME32_3; \ + p+=4; \ + h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; + + switch(len&15) /* or switch(bEnd - p) */ + { + case 12: PROCESS4; + /* fallthrough */ + case 8: PROCESS4; + /* fallthrough */ + case 4: PROCESS4; + return XXH32_avalanche(h32); + + case 13: PROCESS4; + /* fallthrough */ + case 9: PROCESS4; + /* fallthrough */ + case 5: PROCESS4; + PROCESS1; + return XXH32_avalanche(h32); + + case 14: PROCESS4; + /* fallthrough */ + case 10: PROCESS4; + /* fallthrough */ + case 6: PROCESS4; + PROCESS1; + PROCESS1; + return XXH32_avalanche(h32); + + case 15: PROCESS4; + /* fallthrough */ + case 11: PROCESS4; + /* fallthrough */ + case 7: PROCESS4; + /* fallthrough */ + case 3: PROCESS1; + /* fallthrough */ + case 2: PROCESS1; + /* fallthrough */ + case 1: PROCESS1; + /* fallthrough */ + case 0: return XXH32_avalanche(h32); + } + assert(0); + return h32; /* reaching this point is deemed impossible */ +} + + +FORCE_INLINE U32 +XXH32_endian_align(const void* input, size_t len, U32 seed, + XXH_endianess endian, XXH_alignment align) { const BYTE* p = (const BYTE*)input; const BYTE* bEnd = p + len; U32 h32; -#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) if (p==NULL) { len=0; bEnd=p=(const BYTE*)(size_t)16; @@ -281,7 +364,7 @@ XXH_FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, #endif if (len>=16) { - const BYTE* const limit = bEnd - 16; + const BYTE* const limit = bEnd - 15; U32 v1 = seed + PRIME32_1 + PRIME32_2; U32 v2 = seed + PRIME32_2; U32 v3 = seed + 0; @@ -292,34 +375,17 @@ XXH_FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4; v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4; v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4; - } while (p<=limit); + } while (p < limit); - h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); } else { h32 = seed + PRIME32_5; } - h32 += (U32) len; - - while (p+4<=bEnd) { - h32 += XXH_get32bits(p) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; - p+=4; - } - - while (p<bEnd) { - h32 += (*p) * PRIME32_5; - h32 = XXH_rotl32(h32, 11) * PRIME32_1 ; - p++; - } - - h32 ^= h32 >> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; + h32 += (U32)len; - return h32; + return XXH32_finalize(h32, p, len&15, endian, align); } @@ -371,74 +437,81 @@ XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) { XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */ + memset(&state, 0, sizeof(state)); state.v1 = seed + PRIME32_1 + PRIME32_2; state.v2 = seed + PRIME32_2; state.v3 = seed + 0; state.v4 = seed - PRIME32_1; - memcpy(statePtr, &state, sizeof(state)); + /* do not write into reserved, planned to be removed in a future version */ + memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved)); return XXH_OK; } -XXH_FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) +FORCE_INLINE XXH_errorcode +XXH32_update_endian(XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) { - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; + if (input==NULL) +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) + return XXH_OK; +#else + return XXH_ERROR; #endif - state->total_len_32 += (unsigned)len; - state->large_len |= (len>=16) | (state->total_len_32>=16); + { const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; - if (state->memsize + len < 16) { /* fill in tmp buffer */ - XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); - state->memsize += (unsigned)len; - return XXH_OK; - } + state->total_len_32 += (unsigned)len; + state->large_len |= (len>=16) | (state->total_len_32>=16); - if (state->memsize) { /* some data left from previous update */ - XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); - { const U32* p32 = state->mem32; - state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; - state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; - state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; - state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++; + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); + state->memsize += (unsigned)len; + return XXH_OK; } - p += 16-state->memsize; - state->memsize = 0; - } - - if (p <= bEnd-16) { - const BYTE* const limit = bEnd - 16; - U32 v1 = state->v1; - U32 v2 = state->v2; - U32 v3 = state->v3; - U32 v4 = state->v4; - do { - v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4; - v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4; - v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4; - v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4; - } while (p<=limit); + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const U32* p32 = state->mem32; + state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; + state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; + state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; + state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); + } + p += 16-state->memsize; + state->memsize = 0; + } - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } + if (p <= bEnd-16) { + const BYTE* const limit = bEnd - 16; + U32 v1 = state->v1; + U32 v2 = state->v2; + U32 v3 = state->v3; + U32 v4 = state->v4; + + do { + v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4; + v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4; + v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4; + v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } - if (p < bEnd) { - XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); - state->memsize = (unsigned)(bEnd-p); + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } } return XXH_OK; } + XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) { XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; @@ -450,40 +523,23 @@ XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* } - -XXH_FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) +FORCE_INLINE U32 +XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) { - const BYTE * p = (const BYTE*)state->mem32; - const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize; U32 h32; if (state->large_len) { - h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); + h32 = XXH_rotl32(state->v1, 1) + + XXH_rotl32(state->v2, 7) + + XXH_rotl32(state->v3, 12) + + XXH_rotl32(state->v4, 18); } else { h32 = state->v3 /* == seed */ + PRIME32_5; } h32 += state->total_len_32; - while (p+4<=bEnd) { - h32 += XXH_readLE32(p, endian) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4; - p+=4; - } - - while (p<bEnd) { - h32 += (*p) * PRIME32_5; - h32 = XXH_rotl32(h32, 11) * PRIME32_1; - p++; - } - - h32 ^= h32 >> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; + return XXH32_finalize(h32, state->mem32, state->memsize, endian, XXH_aligned); } @@ -503,7 +559,7 @@ XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in) /*! Default XXH result types are basic unsigned 32 and 64 bits. * The canonical representation follows human-readable write convention, aka big-endian (large digits first). * These functions allow transformation of hash result into and from its canonical format. -* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs. +* This way, hash values can be written into a file or buffer, remaining comparable across different systems. */ XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) @@ -522,18 +578,21 @@ XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src #ifndef XXH_NO_LONG_LONG /* ******************************************************************* -* 64-bits hash functions +* 64-bit hash functions *********************************************************************/ /*====== Memory access ======*/ #ifndef MEM_MODULE # define MEM_MODULE -# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) # include <stdint.h> typedef uint64_t U64; # else - typedef unsigned long long U64; /* if your compiler doesn't support unsigned long long, replace by another 64-bit type here. Note that xxhash.h will also need to be updated. */ + /* if compiler doesn't support unsigned long long, replace by another 64-bit type */ + typedef unsigned long long U64; # endif #endif @@ -583,7 +642,7 @@ static U64 XXH_swap64 (U64 x) } #endif -XXH_FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) { if (align==XXH_unaligned) return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); @@ -591,7 +650,7 @@ XXH_FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, X return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); } -XXH_FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) +FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) { return XXH_readLE64_align(ptr, endian, XXH_unaligned); } @@ -626,14 +685,137 @@ static U64 XXH64_mergeRound(U64 acc, U64 val) return acc; } -XXH_FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) +static U64 XXH64_avalanche(U64 h64) +{ + h64 ^= h64 >> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + return h64; +} + + +#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) + +static U64 +XXH64_finalize(U64 h64, const void* ptr, size_t len, + XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)ptr; + +#define PROCESS1_64 \ + h64 ^= (*p++) * PRIME64_5; \ + h64 = XXH_rotl64(h64, 11) * PRIME64_1; + +#define PROCESS4_64 \ + h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; \ + p+=4; \ + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + +#define PROCESS8_64 { \ + U64 const k1 = XXH64_round(0, XXH_get64bits(p)); \ + p+=8; \ + h64 ^= k1; \ + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; \ +} + + switch(len&31) { + case 24: PROCESS8_64; + /* fallthrough */ + case 16: PROCESS8_64; + /* fallthrough */ + case 8: PROCESS8_64; + return XXH64_avalanche(h64); + + case 28: PROCESS8_64; + /* fallthrough */ + case 20: PROCESS8_64; + /* fallthrough */ + case 12: PROCESS8_64; + /* fallthrough */ + case 4: PROCESS4_64; + return XXH64_avalanche(h64); + + case 25: PROCESS8_64; + /* fallthrough */ + case 17: PROCESS8_64; + /* fallthrough */ + case 9: PROCESS8_64; + PROCESS1_64; + return XXH64_avalanche(h64); + + case 29: PROCESS8_64; + /* fallthrough */ + case 21: PROCESS8_64; + /* fallthrough */ + case 13: PROCESS8_64; + /* fallthrough */ + case 5: PROCESS4_64; + PROCESS1_64; + return XXH64_avalanche(h64); + + case 26: PROCESS8_64; + /* fallthrough */ + case 18: PROCESS8_64; + /* fallthrough */ + case 10: PROCESS8_64; + PROCESS1_64; + PROCESS1_64; + return XXH64_avalanche(h64); + + case 30: PROCESS8_64; + /* fallthrough */ + case 22: PROCESS8_64; + /* fallthrough */ + case 14: PROCESS8_64; + /* fallthrough */ + case 6: PROCESS4_64; + PROCESS1_64; + PROCESS1_64; + return XXH64_avalanche(h64); + + case 27: PROCESS8_64; + /* fallthrough */ + case 19: PROCESS8_64; + /* fallthrough */ + case 11: PROCESS8_64; + PROCESS1_64; + PROCESS1_64; + PROCESS1_64; + return XXH64_avalanche(h64); + + case 31: PROCESS8_64; + /* fallthrough */ + case 23: PROCESS8_64; + /* fallthrough */ + case 15: PROCESS8_64; + /* fallthrough */ + case 7: PROCESS4_64; + /* fallthrough */ + case 3: PROCESS1_64; + /* fallthrough */ + case 2: PROCESS1_64; + /* fallthrough */ + case 1: PROCESS1_64; + /* fallthrough */ + case 0: return XXH64_avalanche(h64); + } + + /* impossible to reach */ + assert(0); + return 0; /* unreachable, but some compilers complain without it */ +} + +FORCE_INLINE U64 +XXH64_endian_align(const void* input, size_t len, U64 seed, + XXH_endianess endian, XXH_alignment align) { const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; + const BYTE* bEnd = p + len; U64 h64; -#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) if (p==NULL) { len=0; bEnd=p=(const BYTE*)(size_t)32; @@ -666,32 +848,7 @@ XXH_FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, h64 += (U64) len; - while (p+8<=bEnd) { - U64 const k1 = XXH64_round(0, XXH_get64bits(p)); - h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; - } - - if (p+4<=bEnd) { - h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p+=4; - } - - while (p<bEnd) { - h64 ^= (*p) * PRIME64_5; - h64 = XXH_rotl64(h64, 11) * PRIME64_1; - p++; - } - - h64 ^= h64 >> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - - return h64; + return XXH64_finalize(h64, p, len, endian, align); } @@ -741,65 +898,71 @@ XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) { XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */ + memset(&state, 0, sizeof(state)); state.v1 = seed + PRIME64_1 + PRIME64_2; state.v2 = seed + PRIME64_2; state.v3 = seed + 0; state.v4 = seed - PRIME64_1; - memcpy(statePtr, &state, sizeof(state)); + /* do not write into reserved, planned to be removed in a future version */ + memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved)); return XXH_OK; } -XXH_FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) +FORCE_INLINE XXH_errorcode +XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) { - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; + if (input==NULL) +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) + return XXH_OK; +#else + return XXH_ERROR; #endif - state->total_len += len; - - if (state->memsize + len < 32) { /* fill in tmp buffer */ - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); - state->memsize += (U32)len; - return XXH_OK; - } + { const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; - if (state->memsize) { /* tmp buffer is full */ - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); - state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian)); - state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian)); - state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian)); - state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian)); - p += 32-state->memsize; - state->memsize = 0; - } + state->total_len += len; - if (p+32 <= bEnd) { - const BYTE* const limit = bEnd - 32; - U64 v1 = state->v1; - U64 v2 = state->v2; - U64 v3 = state->v3; - U64 v4 = state->v4; + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); + state->memsize += (U32)len; + return XXH_OK; + } - do { - v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8; - v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8; - v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8; - v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8; - } while (p<=limit); + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian)); + state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian)); + state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian)); + state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian)); + p += 32-state->memsize; + state->memsize = 0; + } - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } + if (p+32 <= bEnd) { + const BYTE* const limit = bEnd - 32; + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + do { + v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8; + v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8; + v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8; + v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } - if (p < bEnd) { - XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); - state->memsize = (unsigned)(bEnd-p); + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } } return XXH_OK; @@ -815,10 +978,8 @@ XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* return XXH64_update_endian(state_in, input, len, XXH_bigEndian); } -XXH_FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) +FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) { - const BYTE * p = (const BYTE*)state->mem64; - const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize; U64 h64; if (state->total_len >= 32) { @@ -833,37 +994,12 @@ XXH_FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endian h64 = XXH64_mergeRound(h64, v3); h64 = XXH64_mergeRound(h64, v4); } else { - h64 = state->v3 + PRIME64_5; + h64 = state->v3 /*seed*/ + PRIME64_5; } h64 += (U64) state->total_len; - while (p+8<=bEnd) { - U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian)); - h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; - } - - if (p+4<=bEnd) { - h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p+=4; - } - - while (p<bEnd) { - h64 ^= (*p) * PRIME64_5; - h64 = XXH_rotl64(h64, 11) * PRIME64_1; - p++; - } - - h64 ^= h64 >> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - - return h64; + return XXH64_finalize(h64, state->mem64, (size_t)state->total_len, endian, XXH_aligned); } XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in) diff --git a/lib/xxhash.h b/lib/xxhash.h index 870a6d9..d6bad94 100644 --- a/lib/xxhash.h +++ b/lib/xxhash.h @@ -57,8 +57,8 @@ Q.Score is a measure of quality of the hash function. It depends on successfully passing SMHasher test set. 10 is a perfect score. -A 64-bits version, named XXH64, is available since r35. -It offers much better speed, but for 64-bits applications only. +A 64-bit version, named XXH64, is available since r35. +It offers much better speed, but for 64-bit applications only. Name Speed on 64 bits Speed on 32 bits XXH64 13.8 GB/s 1.9 GB/s XXH32 6.8 GB/s 6.0 GB/s @@ -80,18 +80,19 @@ typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; /* **************************** -* API modifier -******************************/ -/** XXH_PRIVATE_API -* This is useful to include xxhash functions in `static` mode -* in order to inline them, and remove their symbol from the public list. -* Methodology : -* #define XXH_PRIVATE_API -* #include "xxhash.h" -* `xxhash.c` is automatically included. -* It's not useful to compile and link it as a separate module. -*/ -#ifdef XXH_PRIVATE_API + * API modifier + ******************************/ +/** XXH_INLINE_ALL (and XXH_PRIVATE_API) + * This is useful to include xxhash functions in `static` mode + * in order to inline them, and remove their symbol from the public list. + * Inlining can offer dramatic performance improvement on small keys. + * Methodology : + * #define XXH_INLINE_ALL + * #include "xxhash.h" + * `xxhash.c` is automatically included. + * It's not useful to compile and link it as a separate module. + */ +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) # ifndef XXH_STATIC_LINKING_ONLY # define XXH_STATIC_LINKING_ONLY # endif @@ -102,23 +103,24 @@ typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; # elif defined(_MSC_VER) # define XXH_PUBLIC_API static __inline # else -# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */ + /* this version may generate warnings for unused static functions */ +# define XXH_PUBLIC_API static # endif #else # define XXH_PUBLIC_API /* do nothing */ -#endif /* XXH_PRIVATE_API */ - -/*!XXH_NAMESPACE, aka Namespace Emulation : - -If you want to include _and expose_ xxHash functions from within your own library, -but also want to avoid symbol collisions with other libraries which may also include xxHash, - -you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library -with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values). - -Note that no change is required within the calling program as long as it includes `xxhash.h` : -regular symbol name will be automatically translated by this header. -*/ +#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ + +/*! XXH_NAMESPACE, aka Namespace Emulation : + * + * If you want to include _and expose_ xxHash functions from within your own library, + * but also want to avoid symbol collisions with other libraries which may also include xxHash, + * + * you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library + * with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values). + * + * Note that no change is required within the calling program as long as it includes `xxhash.h` : + * regular symbol name will be automatically translated by this header. + */ #ifdef XXH_NAMESPACE # define XXH_CAT(A,B) A##B # define XXH_NAME2(A,B) XXH_CAT(A,B) @@ -149,18 +151,18 @@ regular symbol name will be automatically translated by this header. ***************************************/ #define XXH_VERSION_MAJOR 0 #define XXH_VERSION_MINOR 6 -#define XXH_VERSION_RELEASE 2 +#define XXH_VERSION_RELEASE 5 #define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) XXH_PUBLIC_API unsigned XXH_versionNumber (void); /*-********************************************************************** -* 32-bits hash +* 32-bit hash ************************************************************************/ -typedef unsigned int XXH32_hash_t; +typedef unsigned int XXH32_hash_t; /*! XXH32() : - Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". + Calculate the 32-bit hash of sequence "length" bytes stored at memory address "input". The memory between input & input+length must be valid (allocated and read-accessible). "seed" can be used to alter the result predictably. Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s */ @@ -177,26 +179,25 @@ XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); /* -These functions generate the xxHash of an input provided in multiple segments. -Note that, for small input, they are slower than single-call functions, due to state management. -For small input, prefer `XXH32()` and `XXH64()` . - -XXH state must first be allocated, using XXH*_createState() . - -Start a new hash by initializing state with a seed, using XXH*_reset(). - -Then, feed the hash state by calling XXH*_update() as many times as necessary. -Obviously, input must be allocated and read accessible. -The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. - -Finally, a hash value can be produced anytime, by using XXH*_digest(). -This function returns the nn-bits hash as an int or long long. - -It's still possible to continue inserting input into the hash state after a digest, -and generate some new hashes later on, by calling again XXH*_digest(). - -When done, free XXH state space if it was allocated dynamically. -*/ + * Streaming functions generate the xxHash of an input provided in multiple segments. + * Note that, for small input, they are slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * XXH state must first be allocated, using XXH*_createState() . + * + * Start a new hash by initializing state with a seed, using XXH*_reset(). + * + * Then, feed the hash state by calling XXH*_update() as many times as necessary. + * The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using XXH*_digest(). + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a digest, + * and generate some new hashes later on, by calling again XXH*_digest(). + * + * When done, free XXH state space if it was allocated dynamically. + */ /*====== Canonical representation ======*/ @@ -205,22 +206,22 @@ XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); /* Default result type for XXH functions are primitive unsigned 32 and 64 bits. -* The canonical representation uses human-readable write convention, aka big-endian (large digits first). -* These functions allow transformation of hash result into and from its canonical format. -* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. -*/ + * The canonical representation uses human-readable write convention, aka big-endian (large digits first). + * These functions allow transformation of hash result into and from its canonical format. + * This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. + */ #ifndef XXH_NO_LONG_LONG /*-********************************************************************** -* 64-bits hash +* 64-bit hash ************************************************************************/ typedef unsigned long long XXH64_hash_t; /*! XXH64() : - Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". + Calculate the 64-bit hash of sequence of length "len" stored at memory address "input". "seed" can be used to alter the result predictably. - This function runs faster on 64-bits systems, but slower on 32-bits systems (see benchmark). + This function runs faster on 64-bit systems, but slower on 32-bit systems (see benchmark). */ XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); @@ -241,48 +242,82 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src #endif /* XXH_NO_LONG_LONG */ + #ifdef XXH_STATIC_LINKING_ONLY /* ================================================================================================ - This section contains definitions which are not guaranteed to remain stable. + This section contains declarations which are not guaranteed to remain stable. They may change in future versions, becoming incompatible with a different version of the library. - They shall only be used with static linking. - Never use these definitions in association with dynamic linking ! + These declarations should only be used with static linking. + Never use them in association with dynamic linking ! =================================================================================================== */ -/* These definitions are only meant to allow allocation of XXH state - statically, on stack, or in a struct for example. - Do not use members directly. */ - - struct XXH32_state_s { - unsigned total_len_32; - unsigned large_len; - unsigned v1; - unsigned v2; - unsigned v3; - unsigned v4; - unsigned mem32[4]; /* buffer defined as U32 for alignment */ - unsigned memsize; - unsigned reserved; /* never read nor write, will be removed in a future version */ - }; /* typedef'd to XXH32_state_t */ - -#ifndef XXH_NO_LONG_LONG - struct XXH64_state_s { - unsigned long long total_len; - unsigned long long v1; - unsigned long long v2; - unsigned long long v3; - unsigned long long v4; - unsigned long long mem64[4]; /* buffer defined as U64 for alignment */ - unsigned memsize; - unsigned reserved[2]; /* never read nor write, will be removed in a future version */ - }; /* typedef'd to XXH64_state_t */ +/* These definitions are only present to allow + * static allocation of XXH state, on stack or in a struct for example. + * Never **ever** use members directly. */ + +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include <stdint.h> + +struct XXH32_state_s { + uint32_t total_len_32; + uint32_t large_len; + uint32_t v1; + uint32_t v2; + uint32_t v3; + uint32_t v4; + uint32_t mem32[4]; + uint32_t memsize; + uint32_t reserved; /* never read nor write, might be removed in a future version */ +}; /* typedef'd to XXH32_state_t */ + +struct XXH64_state_s { + uint64_t total_len; + uint64_t v1; + uint64_t v2; + uint64_t v3; + uint64_t v4; + uint64_t mem64[4]; + uint32_t memsize; + uint32_t reserved[2]; /* never read nor write, might be removed in a future version */ +}; /* typedef'd to XXH64_state_t */ + +# else + +struct XXH32_state_s { + unsigned total_len_32; + unsigned large_len; + unsigned v1; + unsigned v2; + unsigned v3; + unsigned v4; + unsigned mem32[4]; + unsigned memsize; + unsigned reserved; /* never read nor write, might be removed in a future version */ +}; /* typedef'd to XXH32_state_t */ + +# ifndef XXH_NO_LONG_LONG /* remove 64-bit support */ +struct XXH64_state_s { + unsigned long long total_len; + unsigned long long v1; + unsigned long long v2; + unsigned long long v3; + unsigned long long v4; + unsigned long long mem64[4]; + unsigned memsize; + unsigned reserved[2]; /* never read nor write, might be removed in a future version */ +}; /* typedef'd to XXH64_state_t */ +# endif + +# endif + + +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# include "xxhash.c" /* include xxhash function bodies as `static`, for inlining */ #endif -# ifdef XXH_PRIVATE_API -# include "xxhash.c" /* include xxhash function bodies as `static`, for inlining */ -# endif - #endif /* XXH_STATIC_LINKING_ONLY */ diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 26a8089..3709f50 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -134,20 +134,19 @@ static int usage_advanced(const char* exeName) DISPLAY( " -r : operate recursively on directories (sets also -m) \n"); #endif DISPLAY( " -l : compress using Legacy format (Linux kernel compression)\n"); - DISPLAY( " -B# : Block size [4-7] (default : 7) \n"); + DISPLAY( " -B# : cut file into blocks of size # bytes [32+] \n"); + DISPLAY( " or predefined block size [4-7] (default: 7) \n"); DISPLAY( " -BD : Block dependency (improve compression ratio) \n"); DISPLAY( " -BX : enable block checksum (default:disabled) \n"); DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled) \n"); DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n"); DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); DISPLAY( "--favor-decSpeed: compressed files decompress faster, but are less compressed \n"); - DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %u)\n", 1); + DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %i)\n", 1); DISPLAY( "Benchmark arguments : \n"); DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n"); DISPLAY( " -e# : test all compression levels from -bX to # (default : 1)\n"); DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s) \n"); - DISPLAY( " -B# : cut file into independent blocks of size # bytes [32+] \n"); - DISPLAY( " or predefined block size [4-7] (default: 7) \n"); if (g_lz4c_legacy_commands) { DISPLAY( "Legacy arguments : \n"); DISPLAY( " -c0 : fast compression \n"); @@ -497,11 +496,12 @@ int main(int argc, const char** argv) DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10)); } else { if (B < 32) badusage(exeName); - BMK_setBlockSize(B); - if (B >= 1024) { - DISPLAYLEVEL(2, "bench: using blocks of size %u KB \n", (U32)(B>>10)); + blockSize = LZ4IO_setBlockSize(B); + BMK_setBlockSize(blockSize); + if (blockSize >= 1024) { + DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10)); } else { - DISPLAYLEVEL(2, "bench: using blocks of size %u bytes \n", (U32)(B)); + DISPLAYLEVEL(2, "using blocks of size %u bytes \n", (U32)(blockSize)); } } break; diff --git a/programs/lz4io.c b/programs/lz4io.c index 28d6537..a35928d 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -110,6 +110,7 @@ static clock_t g_time = 0; static int g_overwrite = 1; static int g_testMode = 0; static int g_blockSizeId = LZ4IO_BLOCKSIZEID_DEFAULT; +static size_t g_blockSize = 0; static int g_blockChecksum = 0; static int g_streamChecksum = 1; static int g_blockIndependence = 1; @@ -178,7 +179,25 @@ size_t LZ4IO_setBlockSizeID(unsigned bsid) static const unsigned maxBlockSizeID = 7; if ((bsid < minBlockSizeID) || (bsid > maxBlockSizeID)) return 0; g_blockSizeId = bsid; - return blockSizeTable[g_blockSizeId-minBlockSizeID]; + g_blockSize = blockSizeTable[g_blockSizeId-minBlockSizeID]; + return g_blockSize; +} + +size_t LZ4IO_setBlockSize(size_t blockSize) +{ + static const size_t minBlockSize = 32; + static const size_t maxBlockSize = 4 MB; + unsigned bsid = 0; + if (blockSize < minBlockSize) blockSize = minBlockSize; + if (blockSize > maxBlockSize) blockSize = maxBlockSize; + g_blockSize = blockSize; + blockSize--; + /* find which of { 64k, 256k, 1MB, 4MB } is closest to blockSize */ + while (blockSize >>= 2) + bsid++; + if (bsid < 7) bsid = 7; + g_blockSizeId = bsid-3; + return g_blockSize; } int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode) @@ -237,7 +256,6 @@ void LZ4IO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); } ** ********************** LZ4 File / Pipe compression ********************* ** ** ************************************************************************ */ -static int LZ4IO_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } static int LZ4IO_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4IO_SKIPPABLEMASK) == LZ4IO_SKIPPABLE0; } @@ -331,42 +349,48 @@ static int LZ4IO_LZ4_compress(const char* src, char* dst, int srcSize, int dstSi * It generates compressed streams using the old 'legacy' format */ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel) { - int (*compressionFunction)(const char* src, char* dst, int srcSize, int dstSize, int cLevel); + typedef int (*compress_f)(const char* src, char* dst, int srcSize, int dstSize, int cLevel); + compress_f const compressionFunction = (compressionlevel < 3) ? LZ4IO_LZ4_compress : LZ4_compress_HC; unsigned long long filesize = 0; unsigned long long compressedfilesize = MAGICNUMBER_SIZE; char* in_buff; char* out_buff; const int outBuffSize = LZ4_compressBound(LEGACY_BLOCKSIZE); - FILE* finput; + FILE* const finput = LZ4IO_openSrcFile(input_filename); FILE* foutput; clock_t clockEnd; /* Init */ clock_t const clockStart = clock(); - compressionFunction = (compressionlevel < 3) ? LZ4IO_LZ4_compress : LZ4_compress_HC; + if (finput == NULL) + EXM_THROW(20, "%s : open file error ", input_filename); - finput = LZ4IO_openSrcFile(input_filename); - if (finput == NULL) EXM_THROW(20, "%s : open file error ", input_filename); foutput = LZ4IO_openDstFile(output_filename); - if (foutput == NULL) { fclose(finput); EXM_THROW(20, "%s : open file error ", input_filename); } + if (foutput == NULL) { + fclose(finput); + EXM_THROW(20, "%s : open file error ", input_filename); + } /* Allocate Memory */ in_buff = (char*)malloc(LEGACY_BLOCKSIZE); - out_buff = (char*)malloc(outBuffSize); - if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory"); + out_buff = (char*)malloc(outBuffSize + 4); + if (!in_buff || !out_buff) + EXM_THROW(21, "Allocation error : not enough memory"); /* Write Archive Header */ LZ4IO_writeLE32(out_buff, LEGACY_MAGICNUMBER); - { size_t const sizeCheck = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput); - if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header"); } + { size_t const writeSize = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput); + if (writeSize != MAGICNUMBER_SIZE) + EXM_THROW(22, "Write error : cannot write header"); + } /* Main Loop */ while (1) { - unsigned int outSize; + int outSize; /* Read Block */ - size_t const inSize = (int) fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput); + size_t const inSize = fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput); + assert(inSize <= LEGACY_BLOCKSIZE); if (inSize == 0) break; - if (inSize > LEGACY_BLOCKSIZE) EXM_THROW(23, "Read error : wrong fread() size report "); /* should be impossible */ filesize += inSize; /* Compress Block */ @@ -376,9 +400,11 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output (int)(filesize>>20), (double)compressedfilesize/filesize*100); /* Write Block */ - LZ4IO_writeLE32(out_buff, outSize); - { size_t const sizeCheck = fwrite(out_buff, 1, outSize+4, foutput); - if (sizeCheck!=(size_t)(outSize+4)) + assert(outSize > 0); + assert(outSize < outBuffSize); + LZ4IO_writeLE32(out_buff, (unsigned)outSize); + { size_t const writeSize = fwrite(out_buff, 1, outSize+4, foutput); + if (writeSize != (size_t)(outSize+4)) EXM_THROW(24, "Write error : cannot write compressed block"); } } if (ferror(finput)) EXM_THROW(25, "Error while reading %s ", input_filename); @@ -491,7 +517,7 @@ static LZ4F_CDict* LZ4IO_createCDict(void) { static cRess_t LZ4IO_createCResources(void) { - const size_t blockSize = (size_t)LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId); + const size_t blockSize = g_blockSize; cRess_t ress; LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&(ress.ctx), LZ4F_VERSION); @@ -535,7 +561,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, void* const srcBuffer = ress.srcBuffer; void* const dstBuffer = ress.dstBuffer; const size_t dstBufferSize = ress.dstBufferSize; - const size_t blockSize = (size_t)LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId); + const size_t blockSize = g_blockSize; size_t readSize; LZ4F_compressionContext_t ctx = ress.ctx; /* just a pointer */ LZ4F_preferences_t prefs; diff --git a/programs/lz4io.h b/programs/lz4io.h index 22c5e3e..33de41f 100644 --- a/programs/lz4io.h +++ b/programs/lz4io.h @@ -78,6 +78,10 @@ int LZ4IO_setTestMode(int yes); return : 0 if error, blockSize if OK */ size_t LZ4IO_setBlockSizeID(unsigned blockSizeID); +/* blockSize : valid values : 32 -> 4MB + return : 0 if error, actual blocksize if OK */ +size_t LZ4IO_setBlockSize(size_t blockSize); + /* Default setting : independent blocks */ typedef enum { LZ4IO_blockLinked=0, LZ4IO_blockIndependent} LZ4IO_blockMode_t; int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode); diff --git a/programs/util.h b/programs/util.h index d74db0d..4aba6a3 100644 --- a/programs/util.h +++ b/programs/util.h @@ -34,6 +34,7 @@ extern "C" { #include <stdlib.h> /* malloc */ #include <string.h> /* strlen, strncpy */ #include <stdio.h> /* fprintf */ +#include <assert.h> #include <sys/types.h> /* stat, utime */ #include <sys/stat.h> /* stat */ #if defined(_MSC_VER) @@ -375,9 +376,9 @@ UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFi * A modified version of realloc(). * If UTIL_realloc() fails the original block is freed. */ -UTIL_STATIC void *UTIL_realloc(void *ptr, size_t size) +UTIL_STATIC void* UTIL_realloc(void* ptr, size_t size) { - void *newptr = realloc(ptr, size); + void* newptr = realloc(ptr, size); if (newptr) return newptr; free(ptr); return NULL; @@ -387,10 +388,10 @@ UTIL_STATIC void *UTIL_realloc(void *ptr, size_t size) #ifdef _WIN32 # define UTIL_HAS_CREATEFILELIST -UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) +UTIL_STATIC int UTIL_prepareFileList(const char* dirName, char** bufStart, size_t* pos, char** bufEnd) { char* path; - int dirLength, fnameLength, pathLength, nbFiles = 0; + int dirLength, nbFiles = 0; WIN32_FIND_DATAA cFile; HANDLE hFile; @@ -411,7 +412,8 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ free(path); do { - fnameLength = (int)strlen(cFile.cFileName); + int pathLength; + int const fnameLength = (int)strlen(cFile.cFileName); path = (char*) malloc(dirLength + fnameLength + 2); if (!path) { FindClose(hFile); return 0; } memcpy(path, dirName, dirLength); @@ -451,12 +453,11 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ # include <dirent.h> /* opendir, readdir */ # include <string.h> /* strerror, memcpy */ -UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) +UTIL_STATIC int UTIL_prepareFileList(const char* dirName, char** bufStart, size_t* pos, char** bufEnd) { - DIR *dir; - struct dirent *entry; - char* path; - int dirLength, fnameLength, pathLength, nbFiles = 0; + DIR* dir; + struct dirent * entry; + int dirLength, nbFiles = 0; if (!(dir = opendir(dirName))) { fprintf(stderr, "Cannot open directory '%s': %s\n", dirName, strerror(errno)); @@ -466,6 +467,8 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ dirLength = (int)strlen(dirName); errno = 0; while ((entry = readdir(dir)) != NULL) { + char* path; + int fnameLength, pathLength; if (strcmp (entry->d_name, "..") == 0 || strcmp (entry->d_name, ".") == 0) continue; fnameLength = (int)strlen(entry->d_name); @@ -508,7 +511,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ #else -UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) +UTIL_STATIC int UTIL_prepareFileList(const char* dirName, char** bufStart, size_t* pos, char** bufEnd) { (void)bufStart; (void)bufEnd; (void)pos; fprintf(stderr, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName); @@ -523,12 +526,13 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ * After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer) * In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called. */ -UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned inputNamesNb, char** allocatedBuffer, unsigned* allocatedNamesNb) +UTIL_STATIC const char** +UTIL_createFileList(const char** inputNames, unsigned inputNamesNb, char** allocatedBuffer, unsigned* allocatedNamesNb) { size_t pos; unsigned i, nbFiles; char* buf = (char*)malloc(LIST_SIZE_INCREASE); - char* bufend = buf + LIST_SIZE_INCREASE; + size_t bufSize = LIST_SIZE_INCREASE; const char** fileTable; if (!buf) return NULL; @@ -536,20 +540,23 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i for (i=0, pos=0, nbFiles=0; i<inputNamesNb; i++) { if (!UTIL_isDirectory(inputNames[i])) { size_t const len = strlen(inputNames[i]); - if (buf + pos + len >= bufend) { - ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE; + if (pos + len >= bufSize) { + size_t newListSize = bufSize + LIST_SIZE_INCREASE; buf = (char*)UTIL_realloc(buf, newListSize); - bufend = buf + newListSize; + bufSize = newListSize; if (!buf) return NULL; } - if (buf + pos + len < bufend) { - strncpy(buf + pos, inputNames[i], bufend - (buf + pos)); + if (pos + len < bufSize) { + strncpy(buf + pos, inputNames[i], bufSize - pos); pos += len + 1; nbFiles++; } } else { + char* bufend = buf + bufSize; nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend); if (buf == NULL) return NULL; + assert(bufend > buf); + bufSize = bufend - buf; } } if (nbFiles == 0) { free(buf); return NULL; } @@ -562,7 +569,7 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i pos += strlen(fileTable[i]) + 1; } - if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; } + if (pos > bufSize) { free(buf); free((void*)fileTable); return NULL; } /* can this happen ? */ *allocatedBuffer = buf; *allocatedNamesNb = nbFiles; @@ -571,7 +578,8 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i } -UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer) +UTIL_STATIC void +UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer) { if (allocatedBuffer) free(allocatedBuffer); if (filenameTable) free((void*)filenameTable); diff --git a/tests/Makefile b/tests/Makefile index 3de111b..24dc581 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -63,7 +63,7 @@ NB_LOOPS ?= -i1 default: all -all: fullbench fuzzer frametest roundTripTest datagen +all: fullbench fuzzer frametest roundTripTest datagen checkFrame all32: CFLAGS+=-m32 all32: all @@ -109,6 +109,9 @@ roundTripTest : lz4.o lz4hc.o xxhash.o roundTripTest.c datagen : $(PRGDIR)/datagen.c datagencli.c $(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT) +checkFrame : lz4frame.o lz4.o lz4hc.o xxhash.o checkFrame.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + clean: @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) @$(MAKE) -C $(PRGDIR) $@ > $(VOID) @@ -118,7 +121,8 @@ clean: fuzzer$(EXT) fuzzer32$(EXT) \ frametest$(EXT) frametest32$(EXT) \ fasttest$(EXT) roundTripTest$(EXT) \ - datagen$(EXT) checkTag$(EXT) + datagen$(EXT) checkTag$(EXT) \ + frameTest$(EXT) @rm -fR $(TESTDIR) @echo Cleaning completed @@ -148,11 +152,17 @@ endif DD:=dd -test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-install +test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-install test-amalgamation test32: CFLAGS+=-m32 test32: test +test-amalgamation: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c + cat $(LZ4DIR)/lz4.c > lz4_all.c + cat $(LZ4DIR)/lz4hc.c >> lz4_all.c + $(CC) -I$(LZ4DIR) -c lz4_all.c + $(RM) lz4_all.c + test-install: lz4 lib liblz4.pc lz4_root=.. ./test_install.sh diff --git a/tests/checkFrame.c b/tests/checkFrame.c new file mode 100644 index 0000000..f7e7a50 --- /dev/null +++ b/tests/checkFrame.c @@ -0,0 +1,306 @@ + /* + checkFrame - verify frame headers + Copyright (C) Yann Collet 2014-2016 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - LZ4 homepage : http://www.lz4.org + - LZ4 source repository : https://github.com/lz4/lz4 + */ + + /*-************************************ + * Compiler specific + **************************************/ + #ifdef _MSC_VER /* Visual Studio */ + # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ + # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ + #endif + + + /*-************************************ + * Includes + **************************************/ + #include "util.h" /* U32 */ + #include <stdlib.h> /* malloc, free */ + #include <stdio.h> /* fprintf */ + #include <string.h> /* strcmp */ + #include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */ + #include <assert.h> + #include "lz4frame.h" /* include multiple times to test correctness/safety */ + #include "lz4frame.h" + #define LZ4F_STATIC_LINKING_ONLY + #include "lz4frame.h" + #include "lz4frame.h" + #include "lz4.h" /* LZ4_VERSION_STRING */ + #define XXH_STATIC_LINKING_ONLY + #include "xxhash.h" /* XXH64 */ + + + /*-************************************ + * Constants + **************************************/ + #define KB *(1U<<10) + #define MB *(1U<<20) + #define GB *(1U<<30) + + + /*-************************************ + * Macros + **************************************/ + #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) + #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } + + /************************************** + * Exceptions + ***************************************/ + #ifndef DEBUG + # define DEBUG 0 + #endif + #define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); + #define EXM_THROW(error, ...) \ +{ \ + DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAYLEVEL(1, "Error %i : ", error); \ + DISPLAYLEVEL(1, __VA_ARGS__); \ + DISPLAYLEVEL(1, " \n"); \ + return(error); \ +} + + + +/*-*************************************** +* Local Parameters +*****************************************/ +static U32 no_prompt = 0; +static U32 displayLevel = 2; +static U32 use_pause = 0; + + +/*-******************************************************* +* Fuzzer functions +*********************************************************/ +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +#define MAX(a,b) ( (a) > (b) ? (a) : (b) ) + +typedef struct { + void* srcBuffer; + size_t srcBufferSize; + void* dstBuffer; + size_t dstBufferSize; + LZ4F_decompressionContext_t ctx; +} cRess_t; + +static int createCResources(cRess_t *ress) +{ + ress->srcBufferSize = 4 MB; + ress->srcBuffer = malloc(ress->srcBufferSize); + ress->dstBufferSize = 4 MB; + ress->dstBuffer = malloc(ress->dstBufferSize); + + if (!ress->srcBuffer || !ress->dstBuffer) { + free(ress->srcBuffer); + free(ress->dstBuffer); + EXM_THROW(20, "Allocation error : not enough memory"); + } + + if (LZ4F_isError( LZ4F_createDecompressionContext(&(ress->ctx), LZ4F_VERSION) )) { + free(ress->srcBuffer); + free(ress->dstBuffer); + EXM_THROW(21, "Unable to create decompression context"); + } + return 0; +} + +static void freeCResources(cRess_t ress) +{ + free(ress.srcBuffer); + free(ress.dstBuffer); + + (void) LZ4F_freeDecompressionContext(ress.ctx); +} + +int frameCheck(cRess_t ress, FILE* const srcFile, unsigned bsid, size_t blockSize) +{ + LZ4F_errorCode_t nextToLoad = 0; + size_t curblocksize = 0; + int partialBlock = 0; + + /* Main Loop */ + for (;;) { + size_t readSize; + size_t pos = 0; + size_t decodedBytes = ress.dstBufferSize; + size_t remaining; + LZ4F_frameInfo_t frameInfo; + + /* Read input */ + readSize = fread(ress.srcBuffer, 1, ress.srcBufferSize, srcFile); + if (!readSize) break; /* reached end of file or stream */ + + while (pos < readSize) { /* still to read */ + /* Decode Input (at least partially) */ + if (!nextToLoad) { + /* LZ4F_decompress returned 0 : starting new frame */ + curblocksize = 0; + remaining = readSize - pos; + nextToLoad = LZ4F_getFrameInfo(ress.ctx, &frameInfo, (char*)(ress.srcBuffer)+pos, &remaining); + if (LZ4F_isError(nextToLoad)) EXM_THROW(22, "Error getting frame info: %s", LZ4F_getErrorName(nextToLoad)); /* XXX */ + if (frameInfo.blockSizeID != bsid) EXM_THROW(23, "Block size ID %u != expected %u", frameInfo.blockSizeID, bsid); + pos += remaining; + /* nextToLoad should be block header size */ + remaining = nextToLoad; + decodedBytes = ress.dstBufferSize; + nextToLoad = LZ4F_decompress(ress.ctx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, NULL); + if (LZ4F_isError(nextToLoad)) EXM_THROW(24, "Decompression error : %s", LZ4F_getErrorName(nextToLoad)); + pos += remaining; + } + decodedBytes = ress.dstBufferSize; + /* nextToLoad should be just enough to cover the next block */ + if (nextToLoad > (readSize - pos)) { + /* block is not fully contained in current buffer */ + partialBlock = 1; + remaining = readSize - pos; + } else { + if (partialBlock) { + partialBlock = 0; + } + remaining = nextToLoad; + } + nextToLoad = LZ4F_decompress(ress.ctx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, NULL); + if (LZ4F_isError(nextToLoad)) EXM_THROW(24, "Decompression error : %s", LZ4F_getErrorName(nextToLoad)); + curblocksize += decodedBytes; + pos += remaining; + if (!partialBlock) { + /* detect small block due to end of frame; the final 4-byte frame checksum could be left in the buffer */ + if ((curblocksize != 0) && (nextToLoad > 4)) { + if (curblocksize != blockSize) + EXM_THROW(25, "Block size %zu != expected %zu, pos %zu\n", curblocksize, blockSize, pos); + } + curblocksize = 0; + } + } + } + /* can be out because readSize == 0, which could be an fread() error */ + if (ferror(srcFile)) EXM_THROW(26, "Read error"); + + if (nextToLoad!=0) EXM_THROW(27, "Unfinished stream"); + + return 0; +} + +int FUZ_usage(const char* programName) +{ + DISPLAY( "Usage :\n"); + DISPLAY( " %s [args] filename\n", programName); + DISPLAY( "\n"); + DISPLAY( "Arguments :\n"); + DISPLAY( " -b# : expected blocksizeID [4-7] (required)\n"); + DISPLAY( " -B# : expected blocksize [32-4194304] (required)\n"); + DISPLAY( " -v : verbose\n"); + DISPLAY( " -h : display help and exit\n"); + return 0; +} + + +int main(int argc, const char** argv) +{ + int argNb; + int bsid=0; + size_t blockSize=0; + const char* const programName = argv[0]; + + /* Check command line */ + for (argNb=1; argNb<argc; argNb++) { + const char* argument = argv[argNb]; + + if(!argument) continue; /* Protection if argument empty */ + + /* Decode command (note : aggregated short commands are allowed) */ + if (argument[0]=='-') { + if (!strcmp(argument, "--no-prompt")) { + no_prompt=1; + displayLevel=1; + continue; + } + argument++; + + while (*argument!=0) { + switch(*argument) + { + case 'h': + return FUZ_usage(programName); + case 'v': + argument++; + displayLevel++; + break; + case 'q': + argument++; + displayLevel--; + break; + case 'p': /* pause at the end */ + argument++; + use_pause = 1; + break; + + case 'b': + argument++; + bsid=0; + while ((*argument>='0') && (*argument<='9')) { + bsid *= 10; + bsid += *argument - '0'; + argument++; + } + break; + + case 'B': + argument++; + blockSize=0; + while ((*argument>='0') && (*argument<='9')) { + blockSize *= 10; + blockSize += *argument - '0'; + argument++; + } + break; + + default: + ; + return FUZ_usage(programName); + } + } + } else { + int err; + FILE *srcFile; + cRess_t ress; + if (bsid == 0 || blockSize == 0) + return FUZ_usage(programName); + DISPLAY("Starting frame checker (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION_STRING); + err = createCResources(&ress); + if (err) return (err); + srcFile = fopen(argument, "rb"); + if ( srcFile==NULL ) { + freeCResources(ress); + EXM_THROW(1, "%s: %s \n", argument, strerror(errno)); + } + err = frameCheck(ress, srcFile, bsid, blockSize); + freeCResources(ress); + fclose(srcFile); + return (err); + } + } + return 0; +} diff --git a/tests/frametest.c b/tests/frametest.c index 4efeb6f..6d2cdd0 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -343,9 +343,10 @@ int basicTests(U32 seed, double compressibility) op += oSize; ip += iSize; } - { U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); - if (crcDest != crcOrig) goto _output_error; } - DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-ostart), COMPRESSIBLE_NOISE_LENGTH); + { U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); + if (crcDest != crcOrig) goto _output_error; + } + DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-ostart), (unsigned)COMPRESSIBLE_NOISE_LENGTH); } } @@ -520,7 +521,7 @@ int basicTests(U32 seed, double compressibility) LZ4F_CDict* const cdict = LZ4F_createCDict(CNBuffer, dictSize); if (cdict == NULL) goto _output_error; CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) ); - + DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with NULL dict : "); CHECK_V(cSizeNoDict, LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity, @@ -840,8 +841,8 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi size_t const iSize = MIN(sampleMax, (size_t)(iend-ip)); size_t const oSize = LZ4F_compressBound(iSize, prefsPtr); cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1); - DISPLAYLEVEL(6, "Sending %zi bytes to compress (stableSrc:%u) \n", - iSize, cOptions.stableSrc); + DISPLAYLEVEL(6, "Sending %u bytes to compress (stableSrc:%u) \n", + (unsigned)iSize, cOptions.stableSrc); result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions); CHECK(LZ4F_isError(result), "Compression failed (error %i : %s)", (int)result, LZ4F_getErrorName(result)); @@ -856,8 +857,18 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi } } } CHECK(op>=oend, "LZ4F_compressFrameBound overflow"); - result = LZ4F_compressEnd(cCtx, op, oend-op, &cOptions); - CHECK(LZ4F_isError(result), "Compression completion failed (error %i : %s)", (int)result, LZ4F_getErrorName(result)); + { size_t const dstEndSafeSize = LZ4F_compressBound(0, prefsPtr); + int const tooSmallDstEnd = ((FUZ_rand(&randState) & 31) == 3); + size_t const dstEndTooSmallSize = (FUZ_rand(&randState) % dstEndSafeSize) + 1; + size_t const dstEndSize = tooSmallDstEnd ? dstEndTooSmallSize : dstEndSafeSize; + BYTE const canaryByte = (BYTE)(FUZ_rand(&randState) & 255); + op[dstEndSize] = canaryByte; + result = LZ4F_compressEnd(cCtx, op, dstEndSize, &cOptions); + CHECK(op[dstEndSize] != canaryByte, "LZ4F_compressEnd writes beyond dstCapacity !"); + if (LZ4F_isError(result)) { + if (tooSmallDstEnd) /* failure is allowed */ continue; + CHECK(1, "Compression completion failed (error %i : %s)", (int)result, LZ4F_getErrorName(result)); + } } op += result; cSize = op-(BYTE*)compressedBuffer; DISPLAYLEVEL(5, "\nCompressed %u bytes into %u \n", (U32)srcSize, (U32)cSize); diff --git a/tests/fullbench.c b/tests/fullbench.c index fd1202d..4b4e921 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -367,7 +367,6 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) size_t readSize; int compressedBuffSize; U32 crcOriginal; - size_t errorCode; /* Check file existence */ if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } @@ -561,7 +560,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) case 8: decompressionFunction = local_LZ4_decompress_safe_forceExtDict; dName = "LZ4_decompress_safe_forceExtDict"; break; #endif case 9: decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress"; - errorCode = LZ4F_compressFrame(compressed_buff, compressedBuffSize, orig_buff, benchedSize, NULL); + { size_t const errorCode = LZ4F_compressFrame(compressed_buff, compressedBuffSize, orig_buff, benchedSize, NULL); if (LZ4F_isError(errorCode)) { DISPLAY("Error while preparing compressed frame\n"); free(orig_buff); @@ -573,6 +572,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) chunkP[0].compressedSize = (int)errorCode; nbChunks = 1; break; + } default : continue; /* skip if unknown ID */ } diff --git a/tests/fuzzer.c b/tests/fuzzer.c index b29e82e..1d23e1a 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -744,6 +744,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c LZ4_attach_dictionary(&LZ4_stream, &LZ4dict); blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue using extDictCtx failed"); + FUZ_CHECKTEST(LZ4_stream.internal_donotuse.dirty, "context should be good"); /* In the future, it might be desirable to let extDictCtx mode's * output diverge from the output generated by regular extDict mode. @@ -754,19 +755,21 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(XXH32(compressedBuffer, blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output"); FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary(), but output buffer is 1 byte too short"); - LZ4_resetStream(&LZ4_stream); + LZ4_resetStream_fast(&LZ4_stream); LZ4_attach_dictionary(&LZ4_stream, &LZ4dict); ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1); FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using extDictCtx should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize); + FUZ_CHECKTEST(!LZ4_stream.internal_donotuse.dirty, "context should be dirty"); FUZ_DISPLAYTEST(); - LZ4_resetStream(&LZ4_stream); + LZ4_resetStream_fast(&LZ4_stream); LZ4_attach_dictionary(&LZ4_stream, &LZ4dict); ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx should work : enough size available within output buffer"); FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output"); FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output"); + FUZ_CHECKTEST(LZ4_stream.internal_donotuse.dirty, "context should be good"); FUZ_DISPLAYTEST(); LZ4_resetStream_fast(&LZ4_stream); @@ -776,6 +779,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx with re-used context should work : enough size available within output buffer"); FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output"); FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output"); + FUZ_CHECKTEST(LZ4_stream.internal_donotuse.dirty, "context should be good"); } /* Decompress with dictionary as external */ @@ -828,17 +832,20 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c LZ4_setCompressionLevel(&LZ4dictHC, compressionLevel-1); blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue failed"); + FUZ_CHECKTEST(LZ4dictHC.internal_donotuse.dirty, "Context should be clean"); FUZ_DISPLAYTEST(); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDict should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize); + FUZ_CHECKTEST(!LZ4dictHC.internal_donotuse.dirty, "Context should be dirty"); FUZ_DISPLAYTEST(); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue should work : enough size available within output buffer"); + FUZ_CHECKTEST(LZ4dictHC.internal_donotuse.dirty, "Context should be clean"); FUZ_DISPLAYTEST(); decodedBuffer[blockSize] = 0; @@ -861,19 +868,22 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC); blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, (int)compressedBufferSize); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue with ExtDictCtx failed"); + FUZ_CHECKTEST(LZ4_streamHC.internal_donotuse.dirty, "Context should be clean"); FUZ_DISPLAYTEST(); - LZ4_resetStreamHC (&LZ4_streamHC, compressionLevel); + LZ4_resetStreamHC_fast (&LZ4_streamHC, compressionLevel); LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC); ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDictCtx should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize); + FUZ_CHECKTEST(!LZ4_streamHC.internal_donotuse.dirty, "Context should be dirty"); FUZ_DISPLAYTEST(); - LZ4_resetStreamHC (&LZ4_streamHC, compressionLevel); + LZ4_resetStreamHC_fast (&LZ4_streamHC, compressionLevel); LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC); ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue using ExtDictCtx should work : enough size available within output buffer"); + FUZ_CHECKTEST(LZ4_streamHC.internal_donotuse.dirty, "Context should be clean"); FUZ_DISPLAYTEST(); LZ4_resetStreamHC_fast (&LZ4_streamHC, compressionLevel); @@ -881,6 +891,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx and fast reset size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue using ExtDictCtx and fast reset should work : enough size available within output buffer"); + FUZ_CHECKTEST(LZ4_streamHC.internal_donotuse.dirty, "Context should be clean"); FUZ_DISPLAYTEST(); decodedBuffer[blockSize] = 0; @@ -990,6 +1001,7 @@ static void FUZ_unitTests(int compressionLevel) LZ4_resetStream(&streamingState); result = LZ4_compress_fast_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1, 1); FUZ_CHECKTEST(result==0, "LZ4_compress_fast_continue() compression failed!"); + FUZ_CHECKTEST(streamingState.internal_donotuse.dirty, "context should be clean") result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize); FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); @@ -1012,7 +1024,7 @@ static void FUZ_unitTests(int compressionLevel) XXH64_reset(&xxhOrig, 0); XXH64_reset(&xxhNewSafe, 0); XXH64_reset(&xxhNewFast, 0); - LZ4_resetStream(&streamingState); + LZ4_resetStream_fast(&streamingState); LZ4_setStreamDecode(&decodeStateSafe, NULL, 0); LZ4_setStreamDecode(&decodeStateFast, NULL, 0); @@ -1065,6 +1077,7 @@ static void FUZ_unitTests(int compressionLevel) LZ4_resetStreamHC(&sHC, compressionLevel); result = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize); FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); @@ -1073,10 +1086,11 @@ static void FUZ_unitTests(int compressionLevel) /* simple dictionary HC compression test */ crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, compressionLevel); + LZ4_resetStreamHC_fast(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, 64 KB); result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 64 KB); FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed"); @@ -1087,12 +1101,14 @@ static void FUZ_unitTests(int compressionLevel) { int result1, result2; int segSize = testCompressedSize / 2; crcOrig = XXH64(testInput + segSize, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, compressionLevel); + LZ4_resetStreamHC_fast(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, segSize); result1 = LZ4_compress_HC_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1); FUZ_CHECKTEST(result1==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result1); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); result2 = LZ4_compress_HC_continue(&sHC, testInput + 2*segSize, testCompressed+result1, segSize, segSize-1); FUZ_CHECKTEST(result2==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result2); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result1, segSize, testInput, segSize); FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 1 failed"); @@ -1104,10 +1120,11 @@ static void FUZ_unitTests(int compressionLevel) /* remote dictionary HC compression test */ crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, compressionLevel); + LZ4_resetStreamHC_fast(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, 32 KB); result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 32 KB); FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe_usingDict() decompression failed following remote dictionary HC compression test"); @@ -1125,7 +1142,7 @@ static void FUZ_unitTests(int compressionLevel) int segSize = (FUZ_rand(&randState) & 8191); int segNb = 1; - LZ4_resetStreamHC(&sHC, compressionLevel); + LZ4_resetStreamHC_fast(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, dict, dictSize); XXH64_reset(&crcOrigState, 0); @@ -1136,6 +1153,7 @@ static void FUZ_unitTests(int compressionLevel) crcOrig = XXH64_digest(&crcOrigState); result = LZ4_compress_HC_continue(&sHC, testInput + segStart, testCompressed, segSize, LZ4_compressBound(segSize)); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); result = LZ4_decompress_safe_usingDict(testCompressed, dst, result, segSize, dict, dictSize); FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %i failed", segNb); @@ -1172,7 +1190,7 @@ static void FUZ_unitTests(int compressionLevel) XXH64_reset(&xxhOrig, 0); XXH64_reset(&xxhNewSafe, 0); XXH64_reset(&xxhNewFast, 0); - LZ4_resetStreamHC(&sHC, compressionLevel); + LZ4_resetStreamHC_fast(&sHC, compressionLevel); LZ4_setStreamDecode(&decodeStateSafe, NULL, 0); LZ4_setStreamDecode(&decodeStateFast, NULL, 0); @@ -1183,6 +1201,7 @@ static void FUZ_unitTests(int compressionLevel) memcpy (ringBuffer + rNext, testInput + iNext, messageSize); compressedSize = LZ4_compress_HC_continue(&sHC, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed"); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, messageSize); FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe_continue() test failed"); @@ -1233,7 +1252,7 @@ static void FUZ_unitTests(int compressionLevel) XXH64_reset(&xxhOrig, 0); XXH64_reset(&xxhNewSafe, 0); XXH64_reset(&xxhNewFast, 0); - LZ4_resetStreamHC(&sHC, compressionLevel); + LZ4_resetStreamHC_fast(&sHC, compressionLevel); LZ4_setStreamDecode(&decodeStateSafe, NULL, 0); LZ4_setStreamDecode(&decodeStateFast, NULL, 0); @@ -1246,6 +1265,7 @@ static void FUZ_unitTests(int compressionLevel) compressedSize = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed"); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, ringBufferSafe + dNext, compressedSize, messageSize); FUZ_CHECKTEST(result!=messageSize, "64K D.ringBuffer : LZ4_decompress_safe_continue() test failed"); @@ -1277,6 +1297,7 @@ static void FUZ_unitTests(int compressionLevel) compressedSize = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed"); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); DISPLAYLEVEL(5, "compressed %i bytes to %i bytes \n", messageSize, compressedSize); /* test LZ4_decompress_safe_continue */ diff --git a/tests/test_custom_block_sizes.sh b/tests/test_custom_block_sizes.sh new file mode 100755 index 0000000..aba6733 --- /dev/null +++ b/tests/test_custom_block_sizes.sh @@ -0,0 +1,72 @@ +#/usr/bin/env sh +set -e + +LZ4=../lz4 +CHECKFRAME=./checkFrame +DATAGEN=./datagen + +failures="" + +TMPFILE=/tmp/test_custom_block_sizes.$$ +TMPFILE1=/tmp/test_custom_block_sizes1.$$ +TMPFILE2=/tmp/test_custom_block_sizes2.$$ +$DATAGEN -g12345678 > $TMPFILE1 +$DATAGEN -g12345678 > $TMPFILE2 + +echo Testing -B31 +$LZ4 -f -B31 $TMPFILE1 && failures="31 (should fail) " + +for blocksize in 32 65535 65536 +do + echo Testing -B$blocksize + $LZ4 -f -B$blocksize $TMPFILE1 + $LZ4 -f -B$blocksize $TMPFILE2 + cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4 + $CHECKFRAME -B$blocksize -b4 $TMPFILE.lz4 || failures="$failures $blocksize " +done + +for blocksize in 65537 262143 262144 +do + echo Testing -B$blocksize + $LZ4 -f -B$blocksize $TMPFILE1 + $LZ4 -f -B$blocksize $TMPFILE2 + cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4 + $CHECKFRAME -B$blocksize -b5 $TMPFILE.lz4 || failures="$failures $blocksize " +done + +for blocksize in 262145 1048575 1048576 +do + echo Testing -B$blocksize + $LZ4 -f -B$blocksize $TMPFILE1 + $LZ4 -f -B$blocksize $TMPFILE2 + cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4 + $CHECKFRAME -B$blocksize -b6 $TMPFILE.lz4 || failures="$failures $blocksize " +done + +for blocksize in 1048577 4194303 4194304 +do + echo Testing -B$blocksize + $LZ4 -f -B$blocksize $TMPFILE1 + $LZ4 -f -B$blocksize $TMPFILE2 + cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4 + $CHECKFRAME -B$blocksize -b7 $TMPFILE.lz4 || failures="$failures $blocksize " +done + +for blocksize in 4194305 10485760 +do + echo Testing -B$blocksize + $LZ4 -f -B$blocksize $TMPFILE1 + $LZ4 -f -B$blocksize $TMPFILE2 + cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4 + $CHECKFRAME -B4194304 -b7 $TMPFILE.lz4 || failures="$failures $blocksize " +done + +rm $TMPFILE.lz4 $TMPFILE1 $TMPFILE1.lz4 $TMPFILE2 $TMPFILE2.lz4 +if [ "$failures" == "" ] +then + echo ---- All tests passed + exit 0 +else + echo ---- The following tests had failures: $failures + exit 1 +fi |
