diff options
47 files changed, 1620 insertions, 1267 deletions
diff --git a/.travis.yml b/.travis.yml index dc61505..3353dee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ matrix: # Container-based 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - os: linux sudo: false - env: Ubu=12.04cont Cmd='make -C tests test-lz4 test-lz4c test-fasttest test-fullbench' COMPILER=cc + env: Ubu=12.04cont Cmd='make -C tests test-lz4 test-lz4c test-fullbench' COMPILER=cc - os: linux sudo: false @@ -32,7 +32,8 @@ matrix: - libc6-dev-i386 - gcc-multilib - - env: Ubu=14.04 Cmd='make usan' COMPILER=clang + # presume clang >= v3.9.0 + - env: Ubu=14.04 Cmd='make usan MOREFLAGS=-Wcomma -Werror' COMPILER=clang dist: trusty sudo: required addons: @@ -8,6 +8,7 @@ make install # this command may require root access LZ4's `Makefile` supports standard [Makefile conventions], including [staged installs], [redirection], or [command redefinition]. +It is compatible with parallel builds (`-j#`). [Makefile conventions]: https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html [staged installs]: https://www.gnu.org/prep/standards/html_node/DESTDIR.html @@ -57,12 +57,12 @@ all: allmost manuals .PHONY: allmost allmost: lib lz4 examples -.PHONY: lib lib-release -lib lib-release: +.PHONY: lib lib-release liblz4.a +lib lib-release liblz4.a: @$(MAKE) -C $(LZ4DIR) $@ .PHONY: lz4 lz4-release -lz4 : lib +lz4 : liblz4.a lz4-release : lib-release lz4 lz4-release : @$(MAKE) -C $(PRGDIR) $@ @@ -1,3 +1,16 @@ +v1.8.1 +perf : faster and stronger ultra modes (levels 10+) +perf : slightly faster compression and decompression speed +perf : fix bad degenerative case, reported by @c-morgenstern +fix : decompression failed when using a combination of extDict + low memory address (#397), reported and fixed by Julian Scheid (@jscheid) +cli : support for dictionary compression (`-D`), by Felix Handte @felixhandte +cli : fix : `lz4 -d --rm` preserves timestamp (#441) +cli : fix : do not modify /dev/null permission as root, by @aliceatlas +api : `_destSize()` variant supported for all compression levels +build : `make` and `make test` compatible with `-jX`, reported by @mwgamera +build : can control LZ4LIB_VISIBILITY macro, by @mikir +install: fix man page directory (#387), reported by Stuart Cardall (@itoffshore) + v1.8.0 cli : fix : do not modify /dev/null permissions, reported by @Maokaman1 cli : added GNU separator -- specifying that all following arguments are files @@ -82,6 +82,7 @@ make install # this command may require root access LZ4's `Makefile` supports standard [Makefile conventions], including [staged installs], [redirection], or [command redefinition]. +It is compatible with parallel builds (`-j#`). [Makefile conventions]: https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html [staged installs]: https://www.gnu.org/prep/standards/html_node/DESTDIR.html diff --git a/appveyor.yml b/appveyor.yml index 93c1101..056719a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -82,7 +82,7 @@ build_script: ECHO *** && ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% && ECHO *** && - msbuild "visual\VS2010\lz4.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + msbuild "visual\VS2010\lz4.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /p:EnableWholeProgramOptimization=true /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && ECHO *** && ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% && ECHO *** && @@ -19,7 +19,6 @@ test: - make cmake && make clean - make -C tests test-lz4 - make -C tests test-lz4c - - make -C tests test-fasttest - make -C tests test-frametest - make -C tests test-fullbench - make -C tests test-fuzzer && make clean diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html index 9ab1984..4c9c7de 100644 --- a/doc/lz4_manual.html +++ b/doc/lz4_manual.html @@ -60,26 +60,26 @@ <a name="Chapter4"></a><h2>Simple Functions</h2><pre></pre> -<pre><b>int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize); -</b><p> Compresses 'sourceSize' bytes from buffer 'source' - into already allocated 'dest' buffer of size 'maxDestSize'. - Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize). +<pre><b>int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity); +</b><p> Compresses 'srcSize' bytes from buffer 'src' + into already allocated 'dst' buffer of size 'dstCapacity'. + Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). It also runs faster, so it's a recommended setting. - If the function cannot compress 'source' into a more limited 'dest' budget, + If the function cannot compress 'src' into a limited 'dst' budget, compression stops *immediately*, and the function result is zero. - As a consequence, 'dest' content is not valid. - This function never writes outside 'dest' buffer, nor read outside 'source' buffer. - sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE - maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) - return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) - or 0 if compression fails + As a consequence, 'dst' content is not valid. + This function never writes outside 'dst' buffer, nor read outside 'source' buffer. + srcSize : supported max value is LZ4_MAX_INPUT_VALUE + dstCapacity : full or partial size of buffer 'dst' (which must be already allocated) + return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) + or 0 if compression fails </p></pre><BR> -<pre><b>int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); -</b><p> compressedSize : is the precise full size of the compressed block. - maxDecompressedSize : is the size of destination buffer, which must be already allocated. - return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize) - If destination buffer is not large enough, decoding will stop and output an error code (<0). +<pre><b>int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity); +</b><p> compressedSize : is the exact complete size of the compressed block. + dstCapacity : is the size of destination buffer, which must be already allocated. + return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) + If destination buffer is not large enough, decoding will stop and output an error code (negative value). If the source stream is detected malformed, the function will stop decoding and return a negative result. This function is protected against buffer overflow exploits, including malicious data packets. It never writes outside output buffer, nor reads outside input buffer. @@ -97,7 +97,7 @@ or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) </p></pre><BR> -<pre><b>int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration); +<pre><b>int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); </b><p> Same as LZ4_compress_default(), but allows to select an "acceleration" factor. The larger the acceleration value, the faster the algorithm, but also the lesser the compression. It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. @@ -106,45 +106,45 @@ </p></pre><BR> <pre><b>int LZ4_sizeofState(void); -int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration); +int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); </b><p> Same compression function, just using an externally allocated memory space to store compression state. Use LZ4_sizeofState() to know how much memory must be allocated, and allocate it on 8-bytes boundaries (using malloc() typically). Then, provide it as 'void* state' to compression function. </p></pre><BR> -<pre><b>int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize); -</b><p> Reverse the logic, by compressing as much data as possible from 'source' buffer - into already allocated buffer 'dest' of size 'targetDestSize'. - This function either compresses the entire 'source' content into 'dest' if it's large enough, - or fill 'dest' buffer completely with as much data as possible from 'source'. - *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'. - New value is necessarily <= old value. - return : Nb bytes written into 'dest' (necessarily <= targetDestSize) - or 0 if compression fails +<pre><b>int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize); +</b><p> Reverse the logic : compresses as much data as possible from 'src' buffer + into already allocated buffer 'dst' of size 'targetDestSize'. + This function either compresses the entire 'src' content into 'dst' if it's large enough, + or fill 'dst' buffer completely with as much data as possible from 'src'. + *srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'. + New value is necessarily <= old value. + return : Nb bytes written into 'dst' (necessarily <= targetDestSize) + or 0 if compression fails </p></pre><BR> -<pre><b>int LZ4_decompress_fast (const char* source, char* dest, int originalSize); -</b><p> originalSize : is the original and therefore uncompressed size +<pre><b>int LZ4_decompress_fast (const char* src, char* dst, int originalSize); +</b><p> originalSize : is the original uncompressed size return : the number of bytes read from the source buffer (in other words, the compressed size) If the source stream is detected malformed, the function will stop decoding and return a negative result. - Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes. - note : This function fully respect memory boundaries for properly formed compressed data. + Destination buffer must be already allocated. Its size must be >= 'originalSize' bytes. + note : This function respects memory boundaries for *properly formed* compressed data. It is a bit faster than LZ4_decompress_safe(). However, it does not provide any protection against intentionally modified data stream (malicious input). Use this function in trusted environment only (data to decode comes from a trusted source). </p></pre><BR> -<pre><b>int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); -</b><p> This function decompress a compressed block of size 'compressedSize' at position 'source' - into destination buffer 'dest' of size 'maxDecompressedSize'. - The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, - reducing decompression time. - return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) - Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. +<pre><b>int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity); +</b><p> This function decompress a compressed block of size 'srcSize' at position 'src' + into destination buffer 'dst' of size 'dstCapacity'. + The function will decompress a minimum of 'targetOutputSize' bytes, and stop after that. + However, it's not accurate, and may write more than 'targetOutputSize' (but <= dstCapacity). + @return : the number of bytes decoded in the destination buffer (necessarily <= dstCapacity) + Note : this number can be < 'targetOutputSize' should the compressed block contain less data. Always control how many bytes were decoded. If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets + This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets. </p></pre><BR> <a name="Chapter6"></a><h2>Streaming Compression Functions</h2><pre></pre> @@ -158,24 +158,29 @@ int LZ4_freeStream (LZ4_stream_t* streamPtr); <pre><b>void LZ4_resetStream (LZ4_stream_t* streamPtr); </b><p> An LZ4_stream_t structure can be allocated once and re-used multiple times. - Use this function to init an allocated `LZ4_stream_t` structure and start a new compression. + Use this function to start compressing a new stream. </p></pre><BR> <pre><b>int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); -</b><p> Use this function to load a static dictionary into LZ4_stream. +</b><p> Use this function to load a static dictionary into LZ4_stream_t. Any previous data will be forgotten, only 'dictionary' will remain in memory. - Loading a size of 0 is allowed. - Return : dictionary size, in bytes (necessarily <= 64 KB) + Loading a size of 0 is allowed, and is the same as reset. + @return : dictionary size, in bytes (necessarily <= 64 KB) </p></pre><BR> <pre><b>int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); -</b><p> Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio. - Important : Previous data blocks are assumed to remain present and unmodified ! +</b><p> Compress content into 'src' using data from previously compressed blocks, improving compression ratio. 'dst' buffer must be already allocated. If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. - If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function @return==0. + + Important : Up to 64KB of previously compressed data is assumed to remain present and unmodified in memory ! + Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB. + Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. + + @return : size of compressed block + or 0 if there is an error (typically, compressed data cannot fit into 'dst') After an error, the stream status is invalid, it can only be reset or freed. </p></pre><BR> @@ -193,36 +198,41 @@ int LZ4_freeStream (LZ4_stream_t* streamPtr); <pre><b>LZ4_streamDecode_t* LZ4_createStreamDecode(void); int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); -</b><p> creation / destruction of streaming decompression tracking structure +</b><p> creation / destruction of streaming decompression tracking structure. + A tracking structure can be re-used multiple times sequentially. </p></pre><BR> <pre><b>int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); -</b><p> Use this function to instruct where to find the dictionary. - Setting a size of 0 is allowed (same effect as reset). - @return : 1 if OK, 0 if error +</b><p> An LZ4_streamDecode_t structure can be allocated once and re-used multiple times. + Use this function to start decompression of a new stream of blocks. + A dictionary can optionnally be set. Use NULL or size 0 for a simple reset order. + @return : 1 if OK, 0 if error </p></pre><BR> -<pre><b>int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); -int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); -</b><p> These decoding functions allow decompression of multiple blocks in "streaming" mode. - Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) - In the case of a ring buffers, decoding buffer must be either : +<pre><b>int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity); +int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize); +</b><p> These decoding functions allow decompression of consecutive blocks in "streaming" mode. + A block is an unsplittable entity, it must be presented entirely to a decompression function. + Decompression functions only accept one block at a time. + Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB). + + Special : if application sets a ring buffer for decompression, it must respect one of the following conditions : - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. - maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. + maxBlockSize is implementation dependent. It's the maximum size of any single block. In which case, encoding and decoding buffers do not need to be synchronized, and encoding ring buffer can have any size, including small ones ( < 64 KB). - _At least_ 64 KB + 8 bytes + maxBlockSize. In which case, encoding and decoding buffers do not need to be synchronized, and encoding ring buffer can have any size, including larger than decoding buffer. Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, - and indicate where it is saved using LZ4_setStreamDecode() + and indicate where it is saved using LZ4_setStreamDecode() before decompressing next block. </p></pre><BR> -<pre><b>int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); -int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); +<pre><b>int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize); +int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize); </b><p> These decoding functions work the same as a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() They are stand-alone, and don't need an LZ4_streamDecode_t structure. @@ -301,9 +311,11 @@ union LZ4_streamDecode_u { # define LZ4_DEPRECATED(message) </b>/* disable deprecation warnings */<b> #else # define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# if defined (__cplusplus) && (__cplusplus >= 201402) </b>/* C++14 or greater */<b> +# if defined(__clang__) </b>/* clang doesn't handle mixed C++11 and CNU attributes */<b> +# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined (__cplusplus) && (__cplusplus >= 201402) </b>/* C++14 or greater */<b> # define LZ4_DEPRECATED(message) [[deprecated(message)]] -# elif (LZ4_GCC_VERSION >= 405) || defined(__clang__) +# elif (LZ4_GCC_VERSION >= 405) # define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) # elif (LZ4_GCC_VERSION >= 301) # define LZ4_DEPRECATED(message) __attribute__((deprecated)) diff --git a/examples/HCStreaming_ringBuffer.c b/examples/HCStreaming_ringBuffer.c index d49b267..a878577 100644 --- a/examples/HCStreaming_ringBuffer.c +++ b/examples/HCStreaming_ringBuffer.c @@ -1,12 +1,12 @@ // LZ4 HC streaming API example : ring buffer -// Based on previous work from Takayuki Matsuoka +// Based on a previous example by Takayuki Matsuoka /************************************** * Compiler Options **************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS /* for MSVC */ +#if defined(_MSC_VER) && (_MSC_VER <= 1800) /* Visual Studio <= 2013 */ +# define _CRT_SECURE_NO_WARNINGS # define snprintf sprintf_s #endif diff --git a/examples/Makefile b/examples/Makefile index 7647f16..9321c24 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -30,11 +30,11 @@ CPPFLAGS += -I../lib CFLAGS ?= -O3 CFLAGS += -std=gnu99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -FLAGS := $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) +FLAGS := $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) -TESTFILE= Makefile -LZ4DIR := ../lib -LZ4 = ../programs/lz4 +TESTFILE = Makefile +LZ4DIR := ../lib +LZ4 = ../programs/lz4 # Define *.exe as extension for Windows systems @@ -80,14 +80,22 @@ simpleBuffer: $(LZ4DIR)/lz4.c simple_buffer.c $(CC) $(FLAGS) $^ -o $@$(EXT) test : all + @echo "\n=== Print Version ===" ./printVersion$(EXT) + @echo "\n=== Simple compression example ===" + ./simpleBuffer$(EXT) + @echo "\n=== Double-buffer ===" ./doubleBuffer$(EXT) $(TESTFILE) - ./dictionaryRandomAccess$(EXT) $(TESTFILE) $(TESTFILE) 1100 1400 + @echo "\n=== Ring Buffer ===" ./ringBuffer$(EXT) $(TESTFILE) + @echo "\n=== Ring Buffer + LZ4 HC ===" ./ringBufferHC$(EXT) $(TESTFILE) + @echo "\n=== Compress line by line ===" ./lineCompress$(EXT) $(TESTFILE) + @echo "\n=== Dictionary Random Access ===" + ./dictionaryRandomAccess$(EXT) $(TESTFILE) $(TESTFILE) 1100 1400 + @echo "\n=== Frame compression ===" ./frameCompress$(EXT) $(TESTFILE) - ./simpleBuffer$(EXT) $(LZ4) -vt $(TESTFILE).lz4 clean: diff --git a/examples/blockStreaming_doubleBuffer.c b/examples/blockStreaming_doubleBuffer.c index d02f258..acb3455 100644 --- a/examples/blockStreaming_doubleBuffer.c +++ b/examples/blockStreaming_doubleBuffer.c @@ -2,7 +2,7 @@ // Copyright : Takayuki Matsuoka -#ifdef _MSC_VER /* Visual Studio */ +#if defined(_MSC_VER) && (_MSC_VER <= 1800) /* Visual Studio <= 2013 */ # define _CRT_SECURE_NO_WARNINGS # define snprintf sprintf_s #endif diff --git a/examples/blockStreaming_lineByLine.c b/examples/blockStreaming_lineByLine.c index f449aa3..677c426 100644 --- a/examples/blockStreaming_lineByLine.c +++ b/examples/blockStreaming_lineByLine.c @@ -1,8 +1,8 @@ // LZ4 streaming API example : line-by-line logfile compression -// Copyright : Takayuki Matsuoka +// by Takayuki Matsuoka -#ifdef _MSC_VER /* Visual Studio */ +#if defined(_MSC_VER) && (_MSC_VER <= 1800) /* Visual Studio <= 2013 */ # define _CRT_SECURE_NO_WARNINGS # define snprintf sprintf_s #endif diff --git a/examples/blockStreaming_ringBuffer.c b/examples/blockStreaming_ringBuffer.c index 697d342..0b6a3ce 100644 --- a/examples/blockStreaming_ringBuffer.c +++ b/examples/blockStreaming_ringBuffer.c @@ -1,17 +1,14 @@ -// LZ4 streaming API example : ring buffer -// Based on sample code from Takayuki Matsuoka +/* LZ4 streaming API example : ring buffer + * Based on sample code from Takayuki Matsuoka */ /************************************** * Compiler Options **************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS // for MSVC +#if defined(_MSC_VER) && (_MSC_VER <= 1800) /* Visual Studio <= 2013 */ +# define _CRT_SECURE_NO_WARNINGS # define snprintf sprintf_s #endif -#ifdef __GNUC__ -# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ -#endif /************************************** @@ -27,7 +24,7 @@ enum { MESSAGE_MAX_BYTES = 1024, RING_BUFFER_BYTES = 1024 * 8 + MESSAGE_MAX_BYTES, - DECODE_RING_BUFFER = RING_BUFFER_BYTES + MESSAGE_MAX_BYTES // Intentionally larger, to test unsynchronized ring buffers + DECODE_RING_BUFFER = RING_BUFFER_BYTES + MESSAGE_MAX_BYTES /* Intentionally larger, to test unsynchronized ring buffers */ }; @@ -50,7 +47,7 @@ size_t read_bin(FILE* fp, void* array, int arrayBytes) { void test_compress(FILE* outFp, FILE* inpFp) { - LZ4_stream_t lz4Stream_body = { 0 }; + LZ4_stream_t lz4Stream_body = { { 0 } }; LZ4_stream_t* lz4Stream = &lz4Stream_body; static char inpBuf[RING_BUFFER_BYTES]; @@ -85,24 +82,22 @@ void test_compress(FILE* outFp, FILE* inpFp) void test_decompress(FILE* outFp, FILE* inpFp) { static char decBuf[DECODE_RING_BUFFER]; - int decOffset = 0; - LZ4_streamDecode_t lz4StreamDecode_body = { 0 }; + int decOffset = 0; + LZ4_streamDecode_t lz4StreamDecode_body = { { 0 } }; LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body; for(;;) { int cmpBytes = 0; char cmpBuf[CMPBUFSIZE]; - { - const size_t r0 = read_int32(inpFp, &cmpBytes); + { const size_t r0 = read_int32(inpFp, &cmpBytes); if(r0 != 1 || cmpBytes <= 0) break; const size_t r1 = read_bin(inpFp, cmpBuf, cmpBytes); if(r1 != (size_t) cmpBytes) break; } - { - char* const decPtr = &decBuf[decOffset]; + { char* const decPtr = &decBuf[decOffset]; const int decBytes = LZ4_decompress_safe_continue( lz4StreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES); if(decBytes <= 0) break; @@ -120,7 +115,7 @@ int compare(FILE* f0, FILE* f1) { int result = 0; - while(0 == result) { + while (0 == result) { char b0[65536]; char b1[65536]; const size_t r0 = fread(b0, 1, sizeof(b0), f0); @@ -128,12 +123,9 @@ int compare(FILE* f0, FILE* f1) result = (int) r0 - (int) r1; - if(0 == r0 || 0 == r1) { - break; - } - if(0 == result) { - result = memcmp(b0, b1, r0); - } + if (0 == r0 || 0 == r1) break; + + if (0 == result) result = memcmp(b0, b1, r0); } return result; @@ -146,7 +138,7 @@ int main(int argc, char** argv) char lz4Filename[256] = { 0 }; char decFilename[256] = { 0 }; - if(argc < 2) { + if (argc < 2) { printf("Please specify input filename\n"); return 0; } @@ -160,9 +152,8 @@ int main(int argc, char** argv) printf("dec = [%s]\n", decFilename); // compress - { - FILE* inpFp = fopen(inpFilename, "rb"); - FILE* outFp = fopen(lz4Filename, "wb"); + { FILE* const inpFp = fopen(inpFilename, "rb"); + FILE* const outFp = fopen(lz4Filename, "wb"); test_compress(outFp, inpFp); @@ -171,9 +162,8 @@ int main(int argc, char** argv) } // decompress - { - FILE* inpFp = fopen(lz4Filename, "rb"); - FILE* outFp = fopen(decFilename, "wb"); + { FILE* const inpFp = fopen(lz4Filename, "rb"); + FILE* const outFp = fopen(decFilename, "wb"); test_decompress(outFp, inpFp); @@ -182,12 +172,11 @@ int main(int argc, char** argv) } // verify - { - FILE* inpFp = fopen(inpFilename, "rb"); - FILE* decFp = fopen(decFilename, "rb"); + { FILE* const inpFp = fopen(inpFilename, "rb"); + FILE* const decFp = fopen(decFilename, "rb"); const int cmp = compare(inpFp, decFp); - if(0 == cmp) { + if (0 == cmp) { printf("Verify : OK\n"); } else { printf("Verify : NG\n"); diff --git a/examples/dictionaryRandomAccess.c b/examples/dictionaryRandomAccess.c index 6acf99b..291fd08 100644 --- a/examples/dictionaryRandomAccess.c +++ b/examples/dictionaryRandomAccess.c @@ -1,6 +1,6 @@ // LZ4 API example : Dictionary Random Access -#ifdef _MSC_VER /* Visual Studio */ +#if defined(_MSC_VER) && (_MSC_VER <= 1800) /* Visual Studio <= 2013 */ # define _CRT_SECURE_NO_WARNINGS # define snprintf sprintf_s #endif diff --git a/examples/frameCompress.c b/examples/frameCompress.c index 8712725..d66a8dc 100644 --- a/examples/frameCompress.c +++ b/examples/frameCompress.c @@ -21,17 +21,15 @@ static const LZ4F_preferences_t lz4_preferences = { }; static size_t compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) { - LZ4F_errorCode_t r; + size_t r=1; /* function result; 1 == error, default (early exit) */ LZ4F_compressionContext_t ctx; char *src, *buf = NULL; - size_t size, n, k, count_in = 0, count_out, offset = 0, frame_size; + size_t size, count_in = 0, count_out, offset = 0, frame_size; - r = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); - if (LZ4F_isError(r)) { + if (LZ4F_isError( LZ4F_createCompressionContext(&ctx, LZ4F_VERSION) )) { printf("Failed to create context: error %zu\n", r); return 1; } - r = 1; /* function result; 1 == error, by default (early exit) */ src = malloc(BUF_SIZE); if (!src) { @@ -40,41 +38,45 @@ static size_t compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_o } frame_size = LZ4F_compressBound(BUF_SIZE, &lz4_preferences); - size = frame_size + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE; + size = frame_size + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE; buf = malloc(size); if (!buf) { printf("Not enough memory\n"); goto cleanup; } - n = offset = count_out = LZ4F_compressBegin(ctx, buf, size, &lz4_preferences); - if (LZ4F_isError(n)) { - printf("Failed to start compression: error %zu\n", n); - goto cleanup; + { size_t const headerSize = LZ4F_compressBegin(ctx, buf, size, &lz4_preferences); + if (LZ4F_isError(headerSize)) { + printf("Failed to start compression: error %zu\n", headerSize); + goto cleanup; + } + offset = count_out = headerSize; + printf("Buffer size is %zu bytes, header size %zu bytes\n", size, headerSize); } - printf("Buffer size is %zu bytes, header size %zu bytes\n", size, n); for (;;) { - k = fread(src, 1, BUF_SIZE, in); - if (k == 0) + size_t const readSize = fread(src, 1, BUF_SIZE, in); + if (readSize == 0) break; - count_in += k; + count_in += readSize; - n = LZ4F_compressUpdate(ctx, buf + offset, size - offset, src, k, NULL); - if (LZ4F_isError(n)) { - printf("Compression failed: error %zu\n", n); - goto cleanup; + { size_t const compressedSize = LZ4F_compressUpdate(ctx, buf + offset, size - offset, src, readSize, NULL); + if (LZ4F_isError(compressedSize)) { + printf("Compression failed: error %zu\n", compressedSize); + goto cleanup; + } + offset += compressedSize; + count_out += compressedSize; } - offset += n; - count_out += n; if (size - offset < frame_size + LZ4_FOOTER_SIZE) { + size_t writtenSize; printf("Writing %zu bytes\n", offset); - k = fwrite(buf, 1, offset, out); - if (k < offset) { - if (ferror(out)) + writtenSize = fwrite(buf, 1, offset, out); + if (writtenSize < offset) { + if (ferror(out)) /* note : ferror() must follow fwrite */ printf("Write failed\n"); else printf("Short write\n"); @@ -85,31 +87,31 @@ static size_t compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_o } } - n = LZ4F_compressEnd(ctx, buf + offset, size - offset, NULL); - if (LZ4F_isError(n)) { - printf("Failed to end compression: error %zu\n", n); - goto cleanup; + { size_t const compressedSize = LZ4F_compressEnd(ctx, buf + offset, size - offset, NULL); + if (LZ4F_isError(compressedSize)) { + printf("Failed to end compression: error %zu\n", compressedSize); + goto cleanup; + } + offset += compressedSize; + count_out += compressedSize; } - offset += n; - count_out += n; printf("Writing %zu bytes\n", offset); - - k = fwrite(buf, 1, offset, out); - if (k < offset) { - if (ferror(out)) - printf("Write failed\n"); - else - printf("Short write\n"); - goto cleanup; - } + { size_t const writtenSize = fwrite(buf, 1, offset, out); + if (writtenSize < offset) { + if (ferror(out)) + printf("Write failed\n"); + else + printf("Short write\n"); + goto cleanup; + } } *size_in = count_in; *size_out = count_out; - r = 0; + r = 0; /* success */ + cleanup: - if (ctx) - LZ4F_freeCompressionContext(ctx); + LZ4F_freeCompressionContext(ctx); /* supports free on NULL */ free(src); free(buf); return r; @@ -128,28 +130,27 @@ static size_t get_block_size(const LZ4F_frameInfo_t* info) { } } -static size_t decompress_file(FILE *in, FILE *out) { +static size_t decompress_file(FILE* in, FILE* out) { void* const src = malloc(BUF_SIZE); void* dst = NULL; size_t dstCapacity = 0; - LZ4F_dctx *dctx = NULL; - size_t ret; + LZ4F_dctx* dctx = NULL; + size_t ret = 1; /* Initialization */ if (!src) { perror("decompress_file(src)"); goto cleanup; } - ret = LZ4F_createDecompressionContext(&dctx, 100); - if (LZ4F_isError(ret)) { - printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(ret)); - goto cleanup; - } + { size_t const dctxStatus = LZ4F_createDecompressionContext(&dctx, 100); + if (LZ4F_isError(dctxStatus)) { + printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(dctxStatus)); + goto cleanup; + } } /* Decompression */ - ret = 1; while (ret != 0) { /* Load more input */ size_t srcSize = fread(src, 1, BUF_SIZE, in); - void* srcPtr = src; - void* srcEnd = srcPtr + srcSize; + const void* srcPtr = src; + const void* const srcEnd = srcPtr + srcSize; if (srcSize == 0 || ferror(in)) { printf("Decompress: not enough input or error reading file\n"); goto cleanup; @@ -309,4 +310,6 @@ int main(int argc, const char **argv) { fclose(decFp); fclose(inpFp); } + + return 0; } diff --git a/examples/printVersion.c b/examples/printVersion.c index 8607139..7af318a 100644 --- a/examples/printVersion.c +++ b/examples/printVersion.c @@ -1,5 +1,5 @@ // LZ4 trivial example : print Library version number -// Copyright : Takayuki Matsuoka & Yann Collet +// by Takayuki Matsuoka #include <stdio.h> diff --git a/lib/Makefile b/lib/Makefile index 9abb699..dd33f50 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -47,11 +47,12 @@ BUILD_STATIC:=yes CPPFLAGS+= -DXXH_NAMESPACE=LZ4_ CFLAGS ?= -O3 DEBUGFLAGS:= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ - -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \ - -Wpointer-arith -Wstrict-aliasing=1 + -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \ + -Wundef -Wpointer-arith -Wstrict-aliasing=1 CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) +SRCFILES := $(sort $(wildcard *.c)) # OS X linker doesn't support -soname, and use different extension @@ -83,14 +84,14 @@ all: lib all32: CFLAGS+=-m32 all32: all -liblz4.a: *.c +liblz4.a: $(SRCFILES) ifeq ($(BUILD_STATIC),yes) # can be disabled on command line @echo compiling static library @$(CC) $(CPPFLAGS) $(CFLAGS) -c $^ @$(AR) rcs $@ *.o endif -$(LIBLZ4): *.c +$(LIBLZ4): $(SRCFILES) @echo compiling dynamic library $(LIBVER) ifneq (,$(filter Windows%,$(OS))) @$(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll\$@.dll @@ -116,7 +117,8 @@ clean: ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) DESTDIR ?= -# directory variables : GNU convention prefers lowercase +# directory variables : GNU conventions prefer lowercase +# see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html # support both lower and uppercase (BSD), use uppercase in script prefix ?= /usr/local PREFIX ?= $(prefix) diff --git a/lib/README.md b/lib/README.md index b40442c..fc5d4e9 100644 --- a/lib/README.md +++ b/lib/README.md @@ -1,44 +1,43 @@ LZ4 - Library Files ================================ -The directory contains many files, but depending on project's objectives, +The `/lib` directory contains many files, but depending on project's objectives, not all of them are necessary. #### Minimal LZ4 build The minimum required is **`lz4.c`** and **`lz4.h`**, -which will provide the fast compression and decompression algorithm. +which provides the fast compression and decompression algorithm. +They generate and decode data using [LZ4 block format]. -#### The High Compression variant of LZ4 +#### High Compression variant -For more compression at the cost of compression speed, -the High Compression variant **lz4hc** is available. -It's necessary to add **`lz4hc.c`** and **`lz4hc.h`**. -The variant still depends on regular `lz4` source files. -In particular, the decompression is still provided by `lz4.c`. +For more compression ratio at the cost of compression speed, +the High Compression variant called **lz4hc** is available. +Add files **`lz4hc.c`**, **`lz4hc.h`** and **`lz4opt.h`**. +The variant still depends on regular `lib/lz4.*` source files. -#### Compatibility issues +#### Frame variant, for interoperability -In order to produce files or streams compatible with `lz4` command line utility, +In order to produce compressed data compatible with `lz4` command line utility, it's necessary to encode lz4-compressed blocks using the [official interoperable frame format]. This format is generated and decoded automatically by the **lz4frame** library. -In order to work properly, lz4frame needs lz4 and lz4hc, and also **xxhash**, -which provides error detection. -(_Advanced stuff_ : It's possible to hide xxhash symbols into a local namespace. -This is what `liblz4` does, to avoid symbol duplication -in case a user program would link to several libraries containing xxhash symbols.) +Its public API is described in `lib/lz4frame.h`. +In order to work properly, lz4frame needs all other modules present in `/lib`, +including, lz4 and lz4hc, and also **xxhash**. +So it's necessary to include all `*.c` and `*.h` files present in `/lib`. -#### Advanced API +#### Advanced / Experimental API -A more complex `lz4frame_static.h` is also provided. -It contains definitions which are not guaranteed to remain stable within future versions. -It must be used with static linking ***only***. +A complex API defined in `lz4frame_static.h` contains definitions +which are not guaranteed to remain stable in future versions. +As a consequence, it must be used with static linking ***only***. -#### Using MinGW+MSYS to create DLL +#### Windows : using MinGW+MSYS to create DLL DLL can be created using MinGW+MSYS with the `make liblz4` command. This command creates `dll\liblz4.dll` and the import library `dll\liblz4.lib`. @@ -51,23 +50,24 @@ file it should be linked with `dll\liblz4.dll`. For example: ``` gcc $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\liblz4.dll ``` -The compiled executable will require LZ4 DLL which is available at `dll\liblz4.dll`. +The compiled executable will require LZ4 DLL which is available at `dll\liblz4.dll`. -#### Miscellaneous +#### Miscellaneous Other files present in the directory are not source code. There are : - - LICENSE : contains the BSD license text - - Makefile : script to compile or install lz4 library (static or dynamic) - - liblz4.pc.in : for pkg-config (make install) - - README.md : this file + - `LICENSE` : contains the BSD license text + - `Makefile` : `make` script to compile and install lz4 library (static and dynamic) + - `liblz4.pc.in` : for `pkg-config` (used in `make install`) + - `README.md` : this file [official interoperable frame format]: ../doc/lz4_Frame_format.md +[LZ4 block format]: ../doc/lz4_Block_format.md -#### License +#### License All source material within __lib__ directory are BSD 2-Clause licensed. See [LICENSE](LICENSE) for details. -The license is also repeated at the top of each source file. +The license is also reminded at the top of each source file. @@ -85,6 +85,7 @@ #endif + /*-************************************ * Dependency **************************************/ @@ -101,21 +102,43 @@ # pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ #endif /* _MSC_VER */ -#ifndef FORCE_INLINE +#ifndef LZ4_FORCE_INLINE # ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline +# define LZ4_FORCE_INLINE static __forceinline # else # if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ # ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) +# define LZ4_FORCE_INLINE static inline __attribute__((always_inline)) # else -# define FORCE_INLINE static inline +# define LZ4_FORCE_INLINE static inline # endif # else -# define FORCE_INLINE static +# define LZ4_FORCE_INLINE static # endif /* __STDC_VERSION__ */ # endif /* _MSC_VER */ -#endif /* FORCE_INLINE */ +#endif /* LZ4_FORCE_INLINE */ + +/* LZ4_FORCE_O2_GCC_PPC64LE and LZ4_FORCE_O2_INLINE_GCC_PPC64LE + * Gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy, + * together with a simple 8-byte copy loop as a fall-back path. + * However, this optimization hurts the decompression speed by >30%, + * because the execution does not go to the optimized loop + * for typical compressible data, and all of the preamble checks + * before going to the fall-back path become useless overhead. + * This optimization happens only with the -O3 flag, and -O2 generates + * a simple 8-byte copy loop. + * With gcc on ppc64le, all of the LZ4_decompress_* and LZ4_wildCopy + * functions are annotated with __attribute__((optimize("O2"))), + * and also LZ4_wildCopy is forcibly inlined, so that the O2 attribute + * of LZ4_wildCopy does not affect the compression speed. + */ +#if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) +# define LZ4_FORCE_O2_GCC_PPC64LE __attribute__((optimize("O2"))) +# define LZ4_FORCE_O2_INLINE_GCC_PPC64LE __attribute__((optimize("O2"))) LZ4_FORCE_INLINE +#else +# define LZ4_FORCE_O2_GCC_PPC64LE +# define LZ4_FORCE_O2_INLINE_GCC_PPC64LE static +#endif #if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) # define expect(expr,value) (__builtin_expect ((expr),(value)) ) @@ -253,7 +276,8 @@ static void LZ4_copy8(void* dst, const void* src) } /* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ -static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) +LZ4_FORCE_O2_INLINE_GCC_PPC64LE +void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) { BYTE* d = (BYTE*)dstPtr; const BYTE* s = (const BYTE*)srcPtr; @@ -289,15 +313,24 @@ static const int LZ4_minLength = (MFLIMIT+1); /*-************************************ * Error detection **************************************/ -#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ +#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=1) +# include <assert.h> +#else +# ifndef assert +# define assert(condition) ((void)0) +# endif +#endif + +#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ #if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) # include <stdio.h> -# define DEBUGLOG(l, ...) { \ - if (l<=LZ4_DEBUG) { \ - fprintf(stderr, __FILE__ ": "); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, " \n"); \ +static int g_debuglog_enable = 1; +# define DEBUGLOG(l, ...) { \ + if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \ + fprintf(stderr, __FILE__ ": "); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, " \n"); \ } } #else # define DEBUGLOG(l, ...) {} /* disabled */ @@ -307,7 +340,7 @@ static const int LZ4_minLength = (MFLIMIT+1); /*-************************************ * Common functions **************************************/ -static unsigned LZ4_NbCommonBytes (register reg_t val) +static unsigned LZ4_NbCommonBytes (reg_t val) { if (LZ4_isLittleEndian()) { if (sizeof(val)==8) { @@ -318,7 +351,14 @@ static unsigned LZ4_NbCommonBytes (register reg_t val) # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) return (__builtin_ctzll((U64)val) >> 3); # else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, + 0, 3, 1, 3, 1, 4, 2, 7, + 0, 2, 3, 6, 1, 5, 3, 5, + 1, 3, 4, 4, 2, 5, 6, 7, + 7, 0, 1, 2, 3, 3, 4, 6, + 2, 6, 5, 5, 3, 4, 5, 6, + 7, 1, 2, 4, 6, 4, 4, 5, + 7, 2, 6, 5, 7, 6, 7, 7 }; return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; # endif } else /* 32 bits */ { @@ -329,12 +369,15 @@ static unsigned LZ4_NbCommonBytes (register reg_t val) # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) return (__builtin_ctz((U32)val) >> 3); # else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, + 3, 2, 2, 1, 3, 2, 0, 1, + 3, 3, 1, 2, 2, 2, 2, 0, + 3, 1, 2, 0, 1, 0, 1, 1 }; return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; # endif } } else /* Big Endian CPU */ { - if (sizeof(val)==8) { + if (sizeof(val)==8) { /* 64-bits */ # if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) unsigned long r = 0; _BitScanReverse64( &r, val ); @@ -342,8 +385,11 @@ static unsigned LZ4_NbCommonBytes (register reg_t val) # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) return (__builtin_clzll((U64)val) >> 3); # else + static const U32 by32 = sizeof(val)*4; /* 32 on 64 bits (goal), 16 on 32 bits. + Just to avoid some static analyzer complaining about shift by 32 on 32-bits target. + Note that this code path is never triggered in 32-bits mode. */ unsigned r; - if (!(val>>32)) { r=4; } else { r=0; val>>=32; } + if (!(val>>by32)) { r=4; } else { r=0; val>>=by32; } if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } r += (!val); return r; @@ -366,11 +412,20 @@ static unsigned LZ4_NbCommonBytes (register reg_t val) } #define STEPSIZE sizeof(reg_t) -static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) +LZ4_FORCE_INLINE +unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) { const BYTE* const pStart = pIn; - while (likely(pIn<pInLimit-(STEPSIZE-1))) { + if (likely(pIn < pInLimit-(STEPSIZE-1))) { + reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); + if (!diff) { + pIn+=STEPSIZE; pMatch+=STEPSIZE; + } else { + return LZ4_NbCommonBytes(diff); + } } + + while (likely(pIn < pInLimit-(STEPSIZE-1))) { reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); if (!diff) { pIn+=STEPSIZE; pMatch+=STEPSIZE; continue; } pIn += LZ4_NbCommonBytes(diff); @@ -436,7 +491,7 @@ static U32 LZ4_hash5(U64 sequence, tableType_t const tableType) return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); } -FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) +LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) { if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType); return LZ4_hash4(LZ4_read32(p), tableType); @@ -452,7 +507,7 @@ static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableTy } } -FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) { U32 const h = LZ4_hashPosition(p, tableType); LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); @@ -465,7 +520,7 @@ static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tab { const U16* const hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ } -FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) { U32 const h = LZ4_hashPosition(p, tableType); return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); @@ -474,7 +529,7 @@ FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableTy /** LZ4_compress_generic() : inlined, to ensure branches are decided at compilation time */ -FORCE_INLINE int LZ4_compress_generic( +LZ4_FORCE_INLINE int LZ4_compress_generic( LZ4_stream_t_internal* const cctx, const char* const source, char* const dest, @@ -616,7 +671,11 @@ _next_match: *token += ML_MASK; matchCode -= ML_MASK; LZ4_write32(op, 0xFFFFFFFF); - while (matchCode >= 4*255) op+=4, LZ4_write32(op, 0xFFFFFFFF), matchCode -= 4*255; + while (matchCode >= 4*255) { + op+=4; + LZ4_write32(op, 0xFFFFFFFF); + matchCode -= 4*255; + } op += matchCode / 255; *op++ = (BYTE)(matchCode % 255); } else @@ -940,6 +999,7 @@ LZ4_stream_t* LZ4_createStream(void) void LZ4_resetStream (LZ4_stream_t* LZ4_stream) { + DEBUGLOG(4, "LZ4_resetStream"); MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); } @@ -1100,47 +1160,46 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) * Decompression functions *******************************/ /*! LZ4_decompress_generic() : - * This generic decompression function cover all use cases. - * It shall be instantiated several times, using different sets of directives - * Note that it is important this generic function is really inlined, + * This generic decompression function covers all use cases. + * It shall be instantiated several times, using different sets of directives. + * Note that it is important for performance that this function really get inlined, * in order to remove useless branches during compilation optimization. */ -FORCE_INLINE int LZ4_decompress_generic( - const char* const source, - char* const dest, - int inputSize, - int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */ +LZ4_FORCE_O2_GCC_PPC64LE +LZ4_FORCE_INLINE int LZ4_decompress_generic( + const char* const src, + char* const dst, + int srcSize, + int outputSize, /* If endOnInput==endOnInputSize, this value is `dstCapacity` */ int endOnInput, /* endOnOutputSize, endOnInputSize */ int partialDecoding, /* full, partial */ int targetOutputSize, /* only used if partialDecoding==partial */ int dict, /* noDict, withPrefix64k, usingExtDict */ - const BYTE* const lowPrefix, /* == dest when no prefix */ + const BYTE* const lowPrefix, /* always <= dst, == dst when no prefix */ const BYTE* const dictStart, /* only if dict==usingExtDict */ const size_t dictSize /* note : = 0 if noDict */ ) { - /* Local Variables */ - const BYTE* ip = (const BYTE*) source; - const BYTE* const iend = ip + inputSize; + const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; - BYTE* op = (BYTE*) dest; + BYTE* op = (BYTE*) dst; BYTE* const oend = op + outputSize; BYTE* cpy; BYTE* oexit = op + targetOutputSize; - const BYTE* const lowLimit = lowPrefix - dictSize; const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; - const unsigned dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; - const int dec64table[] = {0, 0, 0, -1, 0, 1, 2, 3}; + const unsigned inc32table[8] = {0, 1, 2, 1, 0, 4, 4, 4}; + const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3}; const int safeDecode = (endOnInput==endOnInputSize); const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); /* Special cases */ - if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ - if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ + if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => just decode everything */ + if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); /* Main Loop : decode sequences */ @@ -1149,8 +1208,27 @@ FORCE_INLINE int LZ4_decompress_generic( const BYTE* match; size_t offset; - /* get literal length */ unsigned const token = *ip++; + + /* shortcut for common case : + * in most circumstances, we expect to decode small matches (<= 18 bytes) separated by few literals (<= 14 bytes). + * this shortcut was tested on x86 and x64, where it improves decoding speed. + * it has not yet been benchmarked on ARM, Power, mips, etc. */ + if (((ip + 14 /*maxLL*/ + 2 /*offset*/ <= iend) + & (op + 14 /*maxLL*/ + 18 /*maxML*/ <= oend)) + & ((token < (15<<ML_BITS)) & ((token & ML_MASK) != 15)) ) { + size_t const ll = token >> ML_BITS; + size_t const off = LZ4_readLE16(ip+ll); + const BYTE* const matchPtr = op + ll - off; /* pointer underflow risk ? */ + if ((off >= 18) /* do not deal with overlapping matches */ & (matchPtr >= lowPrefix)) { + size_t const ml = (token & ML_MASK) + MINMATCH; + memcpy(op, ip, 16); op += ll; ip += ll + 2 /*offset*/; + memcpy(op, matchPtr, 18); op += ml; + continue; + } + } + + /* decode literal length */ if ((length=(token>>ML_BITS)) == RUN_MASK) { unsigned s; do { @@ -1184,7 +1262,7 @@ FORCE_INLINE int LZ4_decompress_generic( /* get offset */ offset = LZ4_readLE16(ip); ip+=2; match = op - offset; - if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside buffers */ + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ LZ4_write32(op, (U32)offset); /* costs ~1%; silence an msan warning when offset==0 */ /* get matchlength */ @@ -1228,14 +1306,13 @@ FORCE_INLINE int LZ4_decompress_generic( /* copy match within block */ cpy = op + length; if (unlikely(offset<8)) { - const int dec64 = dec64table[offset]; op[0] = match[0]; op[1] = match[1]; op[2] = match[2]; op[3] = match[3]; - match += dec32table[offset]; + match += inc32table[offset]; memcpy(op+4, match, 4); - match -= dec64; + match -= dec64table[offset]; } else { LZ4_copy8(op, match); match+=8; } op += 8; @@ -1252,31 +1329,34 @@ FORCE_INLINE int LZ4_decompress_generic( LZ4_copy8(op, match); if (length>16) LZ4_wildCopy(op+8, match+8, cpy); } - op=cpy; /* correction */ + op = cpy; /* correction */ } /* end of decoding */ if (endOnInput) - return (int) (((char*)op)-dest); /* Nb of output bytes decoded */ + return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ else - return (int) (((const char*)ip)-source); /* Nb of input bytes read */ + return (int) (((const char*)ip)-src); /* Nb of input bytes read */ /* Overflow error detected */ _output_error: - return (int) (-(((const char*)ip)-source))-1; + return (int) (-(((const char*)ip)-src))-1; } +LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, (BYTE*)dest, NULL, 0); } +LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0); } +LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_fast(const char* source, char* dest, int originalSize) { return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB); @@ -1322,6 +1402,7 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti If it's not possible, save the relevant part of decoded data into a safe buffer, and indicate where it stands using LZ4_setStreamDecode() */ +LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) { LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; @@ -1348,6 +1429,7 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch return result; } +LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) { LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; @@ -1382,7 +1464,8 @@ Advanced decoding functions : the dictionary must be explicitly provided within parameters */ -FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) +LZ4_FORCE_O2_GCC_PPC64LE +LZ4_FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) { if (dictSize==0) return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0); @@ -1394,17 +1477,20 @@ FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); } +LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) { return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); } +LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) { return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize); } /* debug function */ +LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); @@ -72,20 +72,24 @@ extern "C" { /* * LZ4_DLL_EXPORT : * Enable exporting of functions when building a Windows DLL -* LZ4LIB_API : +* LZ4LIB_VISIBILITY : * Control library symbols visibility. */ +#ifndef LZ4LIB_VISIBILITY +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4LIB_VISIBILITY __attribute__ ((visibility ("default"))) +# else +# define LZ4LIB_VISIBILITY +# endif +#endif #if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) -# define LZ4LIB_API __declspec(dllexport) +# define LZ4LIB_API __declspec(dllexport) LZ4LIB_VISIBILITY #elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) -# define LZ4LIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ -#elif defined(__GNUC__) && (__GNUC__ >= 4) -# define LZ4LIB_API __attribute__ ((__visibility__ ("default"))) +# define LZ4LIB_API __declspec(dllimport) LZ4LIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ #else -# define LZ4LIB_API +# define LZ4LIB_API LZ4LIB_VISIBILITY #endif - /*------ Version ------*/ #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ #define LZ4_VERSION_MINOR 8 /* for new (non-breaking) interface capabilities */ @@ -120,30 +124,30 @@ LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; * Simple Functions **************************************/ /*! LZ4_compress_default() : - Compresses 'sourceSize' bytes from buffer 'source' - into already allocated 'dest' buffer of size 'maxDestSize'. - Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize). + Compresses 'srcSize' bytes from buffer 'src' + into already allocated 'dst' buffer of size 'dstCapacity'. + Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). It also runs faster, so it's a recommended setting. - If the function cannot compress 'source' into a more limited 'dest' budget, + If the function cannot compress 'src' into a limited 'dst' budget, compression stops *immediately*, and the function result is zero. - As a consequence, 'dest' content is not valid. - This function never writes outside 'dest' buffer, nor read outside 'source' buffer. - sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE - maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) - return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) - or 0 if compression fails */ -LZ4LIB_API int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize); + As a consequence, 'dst' content is not valid. + This function never writes outside 'dst' buffer, nor read outside 'source' buffer. + srcSize : supported max value is LZ4_MAX_INPUT_VALUE + dstCapacity : full or partial size of buffer 'dst' (which must be already allocated) + return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) + or 0 if compression fails */ +LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity); /*! LZ4_decompress_safe() : - compressedSize : is the precise full size of the compressed block. - maxDecompressedSize : is the size of destination buffer, which must be already allocated. - return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize) - If destination buffer is not large enough, decoding will stop and output an error code (<0). + compressedSize : is the exact complete size of the compressed block. + dstCapacity : is the size of destination buffer, which must be already allocated. + return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) + If destination buffer is not large enough, decoding will stop and output an error code (negative value). If the source stream is detected malformed, the function will stop decoding and return a negative result. This function is protected against buffer overflow exploits, including malicious data packets. It never writes outside output buffer, nor reads outside input buffer. */ -LZ4LIB_API int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); +LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity); /*-************************************ @@ -172,7 +176,7 @@ LZ4_compress_fast() : An acceleration value of "1" is the same as regular LZ4_compress_default() Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1. */ -LZ4LIB_API int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration); +LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); /*! @@ -183,49 +187,49 @@ LZ4_compress_fast_extState() : Then, provide it as 'void* state' to compression function. */ LZ4LIB_API int LZ4_sizeofState(void); -LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration); +LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); /*! LZ4_compress_destSize() : - Reverse the logic, by compressing as much data as possible from 'source' buffer - into already allocated buffer 'dest' of size 'targetDestSize'. - This function either compresses the entire 'source' content into 'dest' if it's large enough, - or fill 'dest' buffer completely with as much data as possible from 'source'. - *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'. - New value is necessarily <= old value. - return : Nb bytes written into 'dest' (necessarily <= targetDestSize) - or 0 if compression fails + Reverse the logic : compresses as much data as possible from 'src' buffer + into already allocated buffer 'dst' of size 'targetDestSize'. + This function either compresses the entire 'src' content into 'dst' if it's large enough, + or fill 'dst' buffer completely with as much data as possible from 'src'. + *srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'. + New value is necessarily <= old value. + return : Nb bytes written into 'dst' (necessarily <= targetDestSize) + or 0 if compression fails */ -LZ4LIB_API int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize); +LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize); /*! -LZ4_decompress_fast() : - originalSize : is the original and therefore uncompressed size +LZ4_decompress_fast() : (unsafe!!) + originalSize : is the original uncompressed size return : the number of bytes read from the source buffer (in other words, the compressed size) If the source stream is detected malformed, the function will stop decoding and return a negative result. - Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes. - note : This function fully respect memory boundaries for properly formed compressed data. + Destination buffer must be already allocated. Its size must be >= 'originalSize' bytes. + note : This function respects memory boundaries for *properly formed* compressed data. It is a bit faster than LZ4_decompress_safe(). However, it does not provide any protection against intentionally modified data stream (malicious input). Use this function in trusted environment only (data to decode comes from a trusted source). */ -LZ4LIB_API int LZ4_decompress_fast (const char* source, char* dest, int originalSize); +LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize); /*! LZ4_decompress_safe_partial() : - This function decompress a compressed block of size 'compressedSize' at position 'source' - into destination buffer 'dest' of size 'maxDecompressedSize'. - The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, - reducing decompression time. - return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) - Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. + This function decompress a compressed block of size 'srcSize' at position 'src' + into destination buffer 'dst' of size 'dstCapacity'. + The function will decompress a minimum of 'targetOutputSize' bytes, and stop after that. + However, it's not accurate, and may write more than 'targetOutputSize' (but <= dstCapacity). + @return : the number of bytes decoded in the destination buffer (necessarily <= dstCapacity) + Note : this number can be < 'targetOutputSize' should the compressed block contain less data. Always control how many bytes were decoded. If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets + This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets. */ -LZ4LIB_API int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); +LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity); /*-********************************************* @@ -242,24 +246,29 @@ LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); /*! LZ4_resetStream() : * An LZ4_stream_t structure can be allocated once and re-used multiple times. - * Use this function to init an allocated `LZ4_stream_t` structure and start a new compression. + * Use this function to start compressing a new stream. */ LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); /*! LZ4_loadDict() : - * Use this function to load a static dictionary into LZ4_stream. + * Use this function to load a static dictionary into LZ4_stream_t. * Any previous data will be forgotten, only 'dictionary' will remain in memory. - * Loading a size of 0 is allowed. - * Return : dictionary size, in bytes (necessarily <= 64 KB) + * Loading a size of 0 is allowed, and is the same as reset. + * @return : dictionary size, in bytes (necessarily <= 64 KB) */ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); /*! LZ4_compress_fast_continue() : - * Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio. - * Important : Previous data blocks are assumed to remain present and unmodified ! + * Compress content into 'src' using data from previously compressed blocks, improving compression ratio. * 'dst' buffer must be already allocated. * If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. - * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function @return==0. + * + * Important : Up to 64KB of previously compressed data is assumed to remain present and unmodified in memory ! + * Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB. + * Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. + * + * @return : size of compressed block + * or 0 if there is an error (typically, compressed data cannot fit into 'dst') * After an error, the stream status is invalid, it can only be reset or freed. */ LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); @@ -280,35 +289,40 @@ LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dict typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* incomplete type (defined later) */ /*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() : - * creation / destruction of streaming decompression tracking structure */ + * creation / destruction of streaming decompression tracking structure. + * A tracking structure can be re-used multiple times sequentially. */ LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); /*! LZ4_setStreamDecode() : - * Use this function to instruct where to find the dictionary. - * Setting a size of 0 is allowed (same effect as reset). - * @return : 1 if OK, 0 if error + * An LZ4_streamDecode_t structure can be allocated once and re-used multiple times. + * Use this function to start decompression of a new stream of blocks. + * A dictionary can optionnally be set. Use NULL or size 0 for a simple reset order. + * @return : 1 if OK, 0 if error */ LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); /*! LZ4_decompress_*_continue() : - * These decoding functions allow decompression of multiple blocks in "streaming" mode. - * Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) - * In the case of a ring buffers, decoding buffer must be either : + * These decoding functions allow decompression of consecutive blocks in "streaming" mode. + * A block is an unsplittable entity, it must be presented entirely to a decompression function. + * Decompression functions only accept one block at a time. + * Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB). + * + * Special : if application sets a ring buffer for decompression, it must respect one of the following conditions : * - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) * In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). * - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. - * maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. + * maxBlockSize is implementation dependent. It's the maximum size of any single block. * In which case, encoding and decoding buffers do not need to be synchronized, * and encoding ring buffer can have any size, including small ones ( < 64 KB). * - _At least_ 64 KB + 8 bytes + maxBlockSize. * In which case, encoding and decoding buffers do not need to be synchronized, * and encoding ring buffer can have any size, including larger than decoding buffer. * Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, - * and indicate where it is saved using LZ4_setStreamDecode() + * and indicate where it is saved using LZ4_setStreamDecode() before decompressing next block. */ -LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); -LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); +LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity); +LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize); /*! LZ4_decompress_*_usingDict() : @@ -316,8 +330,8 @@ LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecod * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() * They are stand-alone, and don't need an LZ4_streamDecode_t structure. */ -LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); -LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); +LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize); +LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize); /*^********************************************** @@ -419,9 +433,11 @@ union LZ4_streamDecode_u { # define LZ4_DEPRECATED(message) /* disable deprecation warnings */ #else # define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# if defined(__clang__) /* clang doesn't handle mixed C++11 and CNU attributes */ +# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ # define LZ4_DEPRECATED(message) [[deprecated(message)]] -# elif (LZ4_GCC_VERSION >= 405) || defined(__clang__) +# elif (LZ4_GCC_VERSION >= 405) # define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) # elif (LZ4_GCC_VERSION >= 301) # define LZ4_DEPRECATED(message) __attribute__((deprecated)) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 3408708..488ab75 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -322,7 +322,7 @@ size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* preferencesPtr) { LZ4F_cctx_t cctxI; - LZ4_stream_t lz4ctx; + LZ4_stream_t lz4ctx; /* pretty large on stack */ LZ4F_preferences_t prefs; LZ4F_compressOptions_t options; BYTE* const dstStart = (BYTE*) dstBuffer; @@ -504,15 +504,15 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, cctxPtr->prefs = *preferencesPtr; /* Ctx Management */ - { U32 const tableID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */ - if (cctxPtr->lz4CtxLevel < tableID) { + { U32 const ctxTypeID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */ + if (cctxPtr->lz4CtxLevel < ctxTypeID) { FREEMEM(cctxPtr->lz4CtxPtr); if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) cctxPtr->lz4CtxPtr = (void*)LZ4_createStream(); else cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC(); if (cctxPtr->lz4CtxPtr == NULL) return err0r(LZ4F_ERROR_allocation_failed); - cctxPtr->lz4CtxLevel = tableID; + cctxPtr->lz4CtxLevel = ctxTypeID; } } /* Buffer Management */ @@ -1253,29 +1253,32 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy); dctx->tmpInSize += sizeToCopy; srcPtr += sizeToCopy; - if (dctx->tmpInSize < dctx->tmpInTarget) { - nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize; /* rest of header + nextBlockHeader */ - doAnotherStage = 0; /* not enough src data, ask for some more */ - break; - } - { size_t const hSize = LZ4F_decodeHeader(dctx, dctx->header, dctx->tmpInTarget); /* will update dStage appropriately */ - if (LZ4F_isError(hSize)) return hSize; - } + } + if (dctx->tmpInSize < dctx->tmpInTarget) { + nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize; /* rest of header + nextBlockHeader */ + doAnotherStage = 0; /* not enough src data, ask for some more */ break; } + { size_t const hSize = LZ4F_decodeHeader(dctx, dctx->header, dctx->tmpInTarget); /* will update dStage appropriately */ + if (LZ4F_isError(hSize)) return hSize; + } + break; case dstage_init: if (dctx->frameInfo.contentChecksumFlag) XXH32_reset(&(dctx->xxh), 0); /* internal buffers allocation */ - { size_t const bufferNeeded = dctx->maxBlockSize + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB) + 4 /* block checksum */; + { size_t const bufferNeeded = dctx->maxBlockSize + + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB); if (bufferNeeded > dctx->maxBufferSize) { /* tmp buffers too small */ dctx->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/ FREEMEM(dctx->tmpIn); - dctx->tmpIn = (BYTE*)ALLOCATOR(dctx->maxBlockSize); - if (dctx->tmpIn == NULL) return err0r(LZ4F_ERROR_allocation_failed); + dctx->tmpIn = (BYTE*)ALLOCATOR(dctx->maxBlockSize + 4 /* block checksum */); + if (dctx->tmpIn == NULL) + return err0r(LZ4F_ERROR_allocation_failed); FREEMEM(dctx->tmpOutBuffer); dctx->tmpOutBuffer= (BYTE*)ALLOCATOR(bufferNeeded); - if (dctx->tmpOutBuffer== NULL) return err0r(LZ4F_ERROR_allocation_failed); + if (dctx->tmpOutBuffer== NULL) + return err0r(LZ4F_ERROR_allocation_failed); dctx->maxBufferSize = bufferNeeded; } } dctx->tmpInSize = 0; @@ -1299,18 +1302,20 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, if (dctx->dStage == dstage_storeBlockHeader) /* can be skipped */ case dstage_storeBlockHeader: - { size_t sizeToCopy = BHSize - dctx->tmpInSize; - if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; + { size_t const remainingInput = (size_t)(srcEnd - srcPtr); + size_t const wantedData = BHSize - dctx->tmpInSize; + size_t const sizeToCopy = MIN(wantedData, remainingInput); memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy); srcPtr += sizeToCopy; dctx->tmpInSize += sizeToCopy; + if (dctx->tmpInSize < BHSize) { /* not enough input for cBlockSize */ nextSrcSizeHint = BHSize - dctx->tmpInSize; doAnotherStage = 0; break; } selectedIn = dctx->tmpIn; - } + } /* if (dctx->dStage == dstage_storeBlockHeader) */ /* decode block header */ { size_t const nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU; @@ -1401,7 +1406,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, dctx->dStage = dstage_getBlockHeader; /* new block */ break; - case dstage_getCBlock: /* entry from dstage_decodeCBlockSize */ + case dstage_getCBlock: if ((size_t)(srcEnd-srcPtr) < dctx->tmpInTarget) { dctx->tmpInSize = 0; dctx->dStage = dstage_storeCBlock; @@ -1414,7 +1419,9 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, break; case dstage_storeCBlock: - { size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, (size_t)(srcEnd-srcPtr)); + { size_t const wantedData = dctx->tmpInTarget - dctx->tmpInSize; + size_t const inputLeft = (size_t)(srcEnd-srcPtr); + size_t const sizeToCopy = MIN(wantedData, inputLeft); memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy); dctx->tmpInSize += sizeToCopy; srcPtr += sizeToCopy; @@ -1465,7 +1472,6 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, case dstage_decodeCBlock_intoTmp: /* not enough place into dst : decode into tmpOut */ - /* ensure enough place for tmpOut */ if (dctx->frameInfo.blockMode == LZ4F_blockLinked) { if (dctx->dict == dctx->tmpOutBuffer) { @@ -1518,29 +1524,27 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, } case dstage_getSuffix: - { size_t const suffixSize = dctx->frameInfo.contentChecksumFlag * 4; - if (dctx->frameRemainingSize) - return err0r(LZ4F_ERROR_frameSize_wrong); /* incorrect frame size decoded */ - if (suffixSize == 0) { /* frame completed */ - nextSrcSizeHint = 0; - LZ4F_resetDecompressionContext(dctx); - doAnotherStage = 0; - break; - } - if ((srcEnd - srcPtr) < 4) { /* not enough size for entire CRC */ - dctx->tmpInSize = 0; - dctx->dStage = dstage_storeSuffix; - } else { - selectedIn = srcPtr; - srcPtr += 4; - } + if (dctx->frameRemainingSize) + return err0r(LZ4F_ERROR_frameSize_wrong); /* incorrect frame size decoded */ + if (!dctx->frameInfo.contentChecksumFlag) { /* no checksum, frame is completed */ + nextSrcSizeHint = 0; + LZ4F_resetDecompressionContext(dctx); + doAnotherStage = 0; + break; + } + if ((srcEnd - srcPtr) < 4) { /* not enough size for entire CRC */ + dctx->tmpInSize = 0; + dctx->dStage = dstage_storeSuffix; + } else { + selectedIn = srcPtr; + srcPtr += 4; } if (dctx->dStage == dstage_storeSuffix) /* can be skipped */ case dstage_storeSuffix: - { - size_t sizeToCopy = 4 - dctx->tmpInSize; - if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; + { size_t const remainingInput = (size_t)(srcEnd - srcPtr); + size_t const wantedData = 4 - dctx->tmpInSize; + size_t const sizeToCopy = MIN(wantedData, remainingInput); memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy); srcPtr += sizeToCopy; dctx->tmpInSize += sizeToCopy; @@ -1550,7 +1554,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, break; } selectedIn = dctx->tmpIn; - } + } /* if (dctx->dStage == dstage_storeSuffix) */ /* case dstage_checkSuffix: */ /* no direct call, avoid scan-build warning */ { U32 const readCRC = LZ4F_readLE32(selectedIn); @@ -1589,7 +1593,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, break; } selectedIn = dctx->header + 4; - } + } /* if (dctx->dStage == dstage_storeSFrameSize) */ /* case dstage_decodeSFrameSize: */ /* no direct access */ { size_t const SFrameSize = LZ4F_readLE32(selectedIn); @@ -1606,36 +1610,38 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, doAnotherStage = 0; nextSrcSizeHint = dctx->tmpInTarget; if (nextSrcSizeHint) break; /* still more to skip */ + /* frame fully skipped : prepare context for a new frame */ LZ4F_resetDecompressionContext(dctx); break; } } - } - - /* preserve history within tmp if necessary */ - if ( (dctx->frameInfo.blockMode==LZ4F_blockLinked) - && (dctx->dict != dctx->tmpOutBuffer) - && (dctx->dStage != dstage_getFrameHeader) - && (!decompressOptionsPtr->stableDst) - && ((unsigned)(dctx->dStage-1) < (unsigned)(dstage_getSuffix-1)) ) + } /* while (doAnotherStage) */ + + /* preserve history within tmp whenever necessary */ + LZ4F_STATIC_ASSERT((unsigned)dstage_init == 2); + if ( (dctx->frameInfo.blockMode==LZ4F_blockLinked) /* next block will use up to 64KB from previous ones */ + && (dctx->dict != dctx->tmpOutBuffer) /* dictionary is not already within tmp */ + && (!decompressOptionsPtr->stableDst) /* cannot rely on dst data to remain there for next call */ + && ((unsigned)(dctx->dStage)-2 < (unsigned)(dstage_getSuffix)-2) ) /* valid stages : [init ... getSuffix[ */ { if (dctx->dStage == dstage_flushOut) { - size_t preserveSize = dctx->tmpOut - dctx->tmpOutBuffer; + size_t const preserveSize = dctx->tmpOut - dctx->tmpOutBuffer; size_t copySize = 64 KB - dctx->tmpOutSize; const BYTE* oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart; if (dctx->tmpOutSize > 64 KB) copySize = 0; if (copySize > preserveSize) copySize = preserveSize; - memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); + if (copySize > 0) + memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); dctx->dict = dctx->tmpOutBuffer; dctx->dictSize = preserveSize + dctx->tmpOutStart; } else { - size_t newDictSize = dctx->dictSize; - const BYTE* oldDictEnd = dctx->dict + dctx->dictSize; - if ((newDictSize) > 64 KB) newDictSize = 64 KB; + const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize; + size_t const newDictSize = MIN(dctx->dictSize, 64 KB); - memcpy(dctx->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize); + if (newDictSize > 0) + memcpy(dctx->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize); dctx->dict = dctx->tmpOutBuffer; dctx->dictSize = newDictSize; diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h index 1899f8e..a59b94b 100644 --- a/lib/lz4frame_static.h +++ b/lib/lz4frame_static.h @@ -43,7 +43,15 @@ extern "C" { /* lz4frame_static.h should be used solely in the context of static linking. * It contains definitions which are not stable and may change in the future. * Never use it in the context of DLL linking. + * + * Defining LZ4F_PUBLISH_STATIC_FUNCTIONS allows one to override this. Use at + * your own risk. */ +#ifdef LZ4F_PUBLISH_STATIC_FUNCTIONS +#define LZ4FLIB_STATIC_API LZ4FLIB_API +#else +#define LZ4FLIB_STATIC_API +#endif /* --- Dependency --- */ @@ -79,7 +87,7 @@ extern "C" { /* enum list is exposed, to handle specific errors */ typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; -LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult); +LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult); @@ -93,8 +101,8 @@ typedef struct LZ4F_CDict_s LZ4F_CDict; * LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. * LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. * `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict */ -LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize); -void LZ4F_freeCDict(LZ4F_CDict* CDict); +LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize); +LZ4FLIB_STATIC_API void LZ4F_freeCDict(LZ4F_CDict* CDict); /*! LZ4_compressFrame_usingCDict() : @@ -106,10 +114,11 @@ void LZ4F_freeCDict(LZ4F_CDict* CDict); * but it's not recommended, as it's the only way to provide dictID in the frame header. * @return : number of bytes written into dstBuffer. * or an error code if it fails (can be tested using LZ4F_isError()) */ -size_t LZ4F_compressFrame_usingCDict(void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const LZ4F_CDict* cdict, - const LZ4F_preferences_t* preferencesPtr); +LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict( + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* preferencesPtr); /*! LZ4F_compressBegin_usingCDict() : @@ -119,21 +128,23 @@ size_t LZ4F_compressFrame_usingCDict(void* dst, size_t dstCapacity, * however, it's the only way to provide dictID in the frame header. * @return : number of bytes written into dstBuffer for the header, * or an error code (which can be tested using LZ4F_isError()) */ -size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctx, - void* dstBuffer, size_t dstCapacity, - const LZ4F_CDict* cdict, - const LZ4F_preferences_t* prefsPtr); +LZ4FLIB_STATIC_API size_t LZ4F_compressBegin_usingCDict( + LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* prefsPtr); /*! LZ4F_decompress_usingDict() : * Same as LZ4F_decompress(), using a predefined dictionary. * Dictionary is used "in place", without any preprocessing. * It must remain accessible throughout the entire frame decoding. */ -size_t LZ4F_decompress_usingDict(LZ4F_dctx* dctxPtr, - void* dstBuffer, size_t* dstSizePtr, - const void* srcBuffer, size_t* srcSizePtr, - const void* dict, size_t dictSize, - const LZ4F_decompressOptions_t* decompressOptionsPtr); +LZ4FLIB_STATIC_API size_t LZ4F_decompress_usingDict( + LZ4F_dctx* dctxPtr, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const void* dict, size_t dictSize, + const LZ4F_decompressOptions_t* decompressOptionsPtr); #if defined (__cplusplus) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 22eb071..f2c2566 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -49,6 +49,7 @@ /*=== Dependency ===*/ +#define LZ4_HC_STATIC_LINKING_ONLY #include "lz4hc.h" @@ -96,7 +97,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) /* Update chains up to ip (excluded) */ -FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip) +LZ4_FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip) { U16* const chainTable = hc4->chainTable; U32* const hashTable = hc4->hashTable; @@ -116,56 +117,73 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip) hc4->nextToUpdate = target; } - -FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* const hc4, /* Index table will be updated */ - const BYTE* const ip, const BYTE* const iLimit, - const BYTE** matchpos, - const int maxNbAttempts) +/** LZ4HC_countBack() : + * @return : negative value, nb of common bytes before ip/match */ +LZ4_FORCE_INLINE +int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match, + const BYTE* const iMin, const BYTE* const mMin) { - U16* const chainTable = hc4->chainTable; - U32* const HashTable = hc4->hashTable; - const BYTE* const base = hc4->base; - const BYTE* const dictBase = hc4->dictBase; - const U32 dictLimit = hc4->dictLimit; - const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); - U32 matchIndex; - int nbAttempts = maxNbAttempts; - size_t ml = 0; + int back=0; + while ( (ip+back > iMin) + && (match+back > mMin) + && (ip[back-1] == match[back-1])) + back--; + return back; +} - /* HC4 match finder */ - LZ4HC_Insert(hc4, ip); - matchIndex = HashTable[LZ4HC_hashPtr(ip)]; +/* LZ4HC_countPattern() : + * pattern32 must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) */ +static unsigned LZ4HC_countPattern(const BYTE* ip, const BYTE* const iEnd, U32 const pattern32) +{ + const BYTE* const iStart = ip; + reg_t const pattern = (sizeof(pattern)==8) ? (reg_t)pattern32 + (((reg_t)pattern32) << 32) : pattern32; + + while (likely(ip < iEnd-(sizeof(pattern)-1))) { + reg_t const diff = LZ4_read_ARCH(ip) ^ pattern; + if (!diff) { ip+=sizeof(pattern); continue; } + ip += LZ4_NbCommonBytes(diff); + return (unsigned)(ip - iStart); + } - while ((matchIndex>=lowLimit) && (nbAttempts)) { - nbAttempts--; - if (matchIndex >= dictLimit) { - const BYTE* const match = base + matchIndex; - if ( (*(match+ml) == *(ip+ml)) /* can be longer */ - && (LZ4_read32(match) == LZ4_read32(ip)) ) - { - size_t const mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; - if (mlt > ml) { ml = mlt; *matchpos = match; } - } - } else { - const BYTE* const match = dictBase + matchIndex; - if (LZ4_read32(match) == LZ4_read32(ip)) { - size_t mlt; - const BYTE* vLimit = ip + (dictLimit - matchIndex); - if (vLimit > iLimit) vLimit = iLimit; - mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; - if ((ip+mlt == vLimit) && (vLimit < iLimit)) - mlt += LZ4_count(ip+mlt, base+dictLimit, iLimit); - if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */ - } + if (LZ4_isLittleEndian()) { + reg_t patternByte = pattern; + while ((ip<iEnd) && (*ip == (BYTE)patternByte)) { + ip++; patternByte >>= 8; + } + } else { /* big endian */ + U32 bitOffset = (sizeof(pattern)*8) - 8; + while (ip < iEnd) { + BYTE const byte = (BYTE)(pattern >> bitOffset); + if (*ip != byte) break; + ip ++; bitOffset -= 8; } - matchIndex -= DELTANEXTU16(chainTable, matchIndex); } - return (int)ml; + return (unsigned)(ip - iStart); +} + +/* LZ4HC_reverseCountPattern() : + * pattern must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) + * read using natural platform endianess */ +static unsigned LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern) +{ + const BYTE* const iStart = ip; + + while (likely(ip >= iLow+4)) { + if (LZ4_read32(ip-4) != pattern) break; + ip -= 4; + } + { const BYTE* bytePtr = (const BYTE*)(&pattern) + 3; /* works for any endianess */ + while (likely(ip>iLow)) { + if (ip[-1] != *bytePtr) break; + ip--; bytePtr--; + } } + return (unsigned)(iStart - ip); } +typedef enum { rep_untested, rep_not, rep_confirmed } repeat_state_e; -FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( +LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( LZ4HC_CCtx_internal* hc4, const BYTE* const ip, const BYTE* const iLowLimit, @@ -173,67 +191,126 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( int longest, const BYTE** matchpos, const BYTE** startpos, - const int maxNbAttempts) + const int maxNbAttempts, + const int patternAnalysis) { U16* const chainTable = hc4->chainTable; U32* const HashTable = hc4->hashTable; const BYTE* const base = hc4->base; const U32 dictLimit = hc4->dictLimit; const BYTE* const lowPrefixPtr = base + dictLimit; - const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); + const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - MAX_DISTANCE; const BYTE* const dictBase = hc4->dictBase; int const delta = (int)(ip-iLowLimit); int nbAttempts = maxNbAttempts; + U32 const pattern = LZ4_read32(ip); U32 matchIndex; + repeat_state_e repeat = rep_untested; + size_t srcPatternLength = 0; - + DEBUGLOG(7, "LZ4HC_InsertAndGetWiderMatch"); /* First Match */ LZ4HC_Insert(hc4, ip); matchIndex = HashTable[LZ4HC_hashPtr(ip)]; + DEBUGLOG(7, "First match at index %u / %u (lowLimit)", + matchIndex, lowLimit); while ((matchIndex>=lowLimit) && (nbAttempts)) { + DEBUGLOG(7, "remaining attempts : %i", nbAttempts); nbAttempts--; if (matchIndex >= dictLimit) { const BYTE* const matchPtr = base + matchIndex; if (*(iLowLimit + longest) == *(matchPtr - delta + longest)) { - if (LZ4_read32(matchPtr) == LZ4_read32(ip)) { + if (LZ4_read32(matchPtr) == pattern) { int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit); + #if 0 + /* more generic but unfortunately slower on clang */ + int const back = LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr); + #else int back = 0; - while ( (ip+back > iLowLimit) && (matchPtr+back > lowPrefixPtr) && (ip[back-1] == matchPtr[back-1])) { back--; } - + #endif mlt -= back; if (mlt > longest) { longest = mlt; *matchpos = matchPtr+back; *startpos = ip+back; - } } } - } else { + } } + } + } else { /* matchIndex < dictLimit */ const BYTE* const matchPtr = dictBase + matchIndex; - if (LZ4_read32(matchPtr) == LZ4_read32(ip)) { + if (LZ4_read32(matchPtr) == pattern) { int mlt; - int back=0; + int back = 0; const BYTE* vLimit = ip + (dictLimit - matchIndex); if (vLimit > iHighLimit) vLimit = iHighLimit; mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); - while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchPtr[back-1])) back--; + while ( (ip+back > iLowLimit) + && (matchIndex+back > lowLimit) + && (ip[back-1] == matchPtr[back-1])) + back--; mlt -= back; - if (mlt > longest) { longest = mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; } - } - } - matchIndex -= DELTANEXTU16(chainTable, matchIndex); - } + if (mlt > longest) { + longest = mlt; + *matchpos = base + matchIndex + back; + *startpos = ip + back; + } } } + + { U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex); + matchIndex -= nextOffset; + if (patternAnalysis && nextOffset==1) { + /* may be a repeated pattern */ + if (repeat == rep_untested) { + if ( ((pattern & 0xFFFF) == (pattern >> 16)) + & ((pattern & 0xFF) == (pattern >> 24)) ) { + repeat = rep_confirmed; + srcPatternLength = LZ4HC_countPattern(ip+4, iHighLimit, pattern) + 4; + } else { + repeat = rep_not; + } } + if ( (repeat == rep_confirmed) + && (matchIndex >= dictLimit) ) { /* same segment only */ + const BYTE* const matchPtr = base + matchIndex; + if (LZ4_read32(matchPtr) == pattern) { /* good candidate */ + size_t const forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern); + const BYTE* const maxLowPtr = (lowPrefixPtr + MAX_DISTANCE >= ip) ? lowPrefixPtr : ip - MAX_DISTANCE; + size_t const backLength = LZ4HC_reverseCountPattern(matchPtr, maxLowPtr, pattern); + size_t const currentSegmentLength = backLength + forwardPatternLength; + + if ( (currentSegmentLength >= srcPatternLength) /* current pattern segment large enough to contain full srcPatternLength */ + && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */ + matchIndex += (U32)forwardPatternLength - (U32)srcPatternLength; /* best position, full pattern, might be followed by more match */ + } else { + matchIndex -= (U32)backLength; /* let's go to farthest segment position, will find a match of length currentSegmentLength + maybe some back */ + } + } } } } + } /* while ((matchIndex>=lowLimit) && (nbAttempts)) */ return longest; } +LZ4_FORCE_INLINE +int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index table will be updated */ + const BYTE* const ip, const BYTE* const iLimit, + const BYTE** matchpos, + const int maxNbAttempts, + const int patternAnalysis) +{ + const BYTE* uselessPtr = ip; + /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos), + * but this won't be the case here, as we define iLowLimit==ip, + * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */ + return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis); +} + + typedef enum { noLimit = 0, @@ -241,14 +318,10 @@ typedef enum { limitedDestSize = 2, } limitedOutput_directive; -#ifndef LZ4HC_DEBUG -# define LZ4HC_DEBUG 0 -#endif - /* LZ4HC_encodeSequence() : * @return : 0 if ok, * 1 if buffer issue detected */ -FORCE_INLINE int LZ4HC_encodeSequence ( +LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( const BYTE** ip, BYTE** op, const BYTE** anchor, @@ -260,9 +333,21 @@ FORCE_INLINE int LZ4HC_encodeSequence ( size_t length; BYTE* const token = (*op)++; -#if LZ4HC_DEBUG - printf("literal : %u -- match : %u -- offset : %u\n", - (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match)); +#if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 2) + static const BYTE* start = NULL; + static U32 totalCost = 0; + U32 const pos = (start==NULL) ? 0 : (U32)(*anchor - start); + U32 const ll = (U32)(*ip - *anchor); + U32 const llAdd = (ll>=15) ? ((ll-15) / 255) + 1 : 0; + U32 const mlAdd = (matchLength>=19) ? ((matchLength-19) / 255) + 1 : 0; + U32 const cost = 1 + llAdd + ll + 2 + mlAdd; + if (start==NULL) start = *anchor; /* only works for single segment */ + //g_debuglog_enable = (pos >= 2228) & (pos <= 2262); + DEBUGLOG(2, "pos:%7u -- literals:%3u, match:%4i, offset:%5u, cost:%3u + %u", + pos, + (U32)(*ip - *anchor), matchLength, (U32)(*ip-match), + cost, totalCost); + totalCost += cost; #endif /* Encode Literal length */ @@ -285,6 +370,7 @@ FORCE_INLINE int LZ4HC_encodeSequence ( LZ4_writeLE16(*op, (U16)(*ip-match)); *op += 2; /* Encode MatchLength */ + assert(matchLength >= MINMATCH); length = (size_t)(matchLength - MINMATCH); if ((limit) && (*op + (length >> 8) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */ if (length >= ML_MASK) { @@ -319,6 +405,7 @@ static int LZ4HC_compress_hashChain ( ) { const int inputSize = *srcSizePtr; + const int patternAnalysis = (maxNbAttempts > 64); /* levels 8+ */ const BYTE* ip = (const BYTE*) source; const BYTE* anchor = ip; @@ -341,19 +428,13 @@ static int LZ4HC_compress_hashChain ( /* init */ *srcSizePtr = 0; - if (limit == limitedDestSize && maxOutputSize < 1) return 0; /* Impossible to store anything */ - if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ - - ctx->end += inputSize; - if (limit == limitedDestSize) oend -= LASTLITERALS; /* Hack for support limitations LZ4 decompressor */ + if (limit == limitedDestSize) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ if (inputSize < LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ - ip++; - /* Main Loop */ while (ip < mflimit) { - ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref), maxNbAttempts); - if (!ml) { ip++; continue; } + ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis); + if (ml<MINMATCH) { ip++; continue; } /* saved, in case we would skip too much */ start0 = ip; @@ -362,7 +443,9 @@ static int LZ4HC_compress_hashChain ( _Search2: if (ip+ml < mflimit) - ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2, maxNbAttempts); + ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, + ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2, + maxNbAttempts, patternAnalysis); else ml2 = ml; @@ -407,7 +490,9 @@ _Search3: /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */ if (start2 + ml2 < mflimit) - ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, maxNbAttempts); + ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, + start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, + maxNbAttempts, patternAnalysis); else ml3 = ml2; @@ -527,14 +612,6 @@ _dest_overflow: return 0; } -static int LZ4HC_getSearchNum(int compressionLevel) -{ - switch (compressionLevel) { - default: return 0; /* unused */ - case 11: return 128; - case 12: return 1<<10; - } -} static int LZ4HC_compress_generic ( LZ4HC_CCtx_internal* const ctx, @@ -546,24 +623,47 @@ static int LZ4HC_compress_generic ( limitedOutput_directive limit ) { - if (cLevel < 1) cLevel = LZ4HC_CLEVEL_DEFAULT; /* note : convention is different from lz4frame, maybe to reconsider */ - if (cLevel > 9) { - if (limit == limitedDestSize) cLevel = 10; - switch (cLevel) { - case 10: - return LZ4HC_compress_hashChain(ctx, src, dst, srcSizePtr, dstCapacity, 1 << 12, limit); - case 11: - ctx->searchNum = LZ4HC_getSearchNum(cLevel); - return LZ4HC_compress_optimal(ctx, src, dst, *srcSizePtr, dstCapacity, limit, 128, 0); - default: - cLevel = 12; - /* fall-through */ - case 12: - ctx->searchNum = LZ4HC_getSearchNum(cLevel); - return LZ4HC_compress_optimal(ctx, src, dst, *srcSizePtr, dstCapacity, limit, LZ4_OPT_NUM, 1); - } + typedef enum { lz4hc, lz4opt } lz4hc_strat_e; + typedef struct { + lz4hc_strat_e strat; + U32 nbSearches; + U32 targetLength; + } cParams_t; + static const cParams_t clTable[LZ4HC_CLEVEL_MAX+1] = { + { lz4hc, 2, 16 }, /* 0, unused */ + { lz4hc, 2, 16 }, /* 1, unused */ + { lz4hc, 2, 16 }, /* 2, unused */ + { lz4hc, 4, 16 }, /* 3 */ + { lz4hc, 8, 16 }, /* 4 */ + { lz4hc, 16, 16 }, /* 5 */ + { lz4hc, 32, 16 }, /* 6 */ + { lz4hc, 64, 16 }, /* 7 */ + { lz4hc, 128, 16 }, /* 8 */ + { lz4hc, 256, 16 }, /* 9 */ + { lz4opt, 96, 64 }, /*10==LZ4HC_CLEVEL_OPT_MIN*/ + { lz4opt, 512,128 }, /*11 */ + { lz4opt,8192, LZ4_OPT_NUM }, /* 12==LZ4HC_CLEVEL_MAX */ + }; + + if (limit == limitedDestSize && dstCapacity < 1) return 0; /* Impossible to store anything */ + if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size (too large or negative) */ + + ctx->end += *srcSizePtr; + if (cLevel < 1) cLevel = LZ4HC_CLEVEL_DEFAULT; /* note : convention is different from lz4frame, maybe something to review */ + cLevel = MIN(LZ4HC_CLEVEL_MAX, cLevel); + assert(cLevel >= 0); + assert(cLevel <= LZ4HC_CLEVEL_MAX); + { cParams_t const cParam = clTable[cLevel]; + if (cParam.strat == lz4hc) + return LZ4HC_compress_hashChain(ctx, + src, dst, srcSizePtr, dstCapacity, + cParam.nbSearches, limit); + assert(cParam.strat == lz4opt); + return LZ4HC_compress_optimal(ctx, + src, dst, srcSizePtr, dstCapacity, + cParam.nbSearches, cParam.targetLength, limit, + cLevel == LZ4HC_CLEVEL_MAX); /* ultra mode */ } - return LZ4HC_compress_hashChain(ctx, src, dst, srcSizePtr, dstCapacity, 1 << (cLevel-1), limit); /* levels 1-9 */ } @@ -596,8 +696,7 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in } /* LZ4_compress_HC_destSize() : - * currently, only compatible with Hash Chain implementation, - * hence limit compression level to LZ4HC_CLEVEL_OPT_MIN-1*/ + * only compatible with regular HC parser */ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int cLevel) { LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse; @@ -624,18 +723,13 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) { LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ LZ4_streamHCPtr->internal_donotuse.base = NULL; - if (compressionLevel > LZ4HC_CLEVEL_MAX) compressionLevel = LZ4HC_CLEVEL_MAX; /* cap compression level */ - LZ4_streamHCPtr->internal_donotuse.compressionLevel = compressionLevel; - LZ4_streamHCPtr->internal_donotuse.searchNum = LZ4HC_getSearchNum(compressionLevel); + LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); } void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) { - int const currentCLevel = LZ4_streamHCPtr->internal_donotuse.compressionLevel; - int const minCLevel = currentCLevel < LZ4HC_CLEVEL_OPT_MIN ? 1 : LZ4HC_CLEVEL_OPT_MIN; - int const maxCLevel = currentCLevel < LZ4HC_CLEVEL_OPT_MIN ? LZ4HC_CLEVEL_OPT_MIN-1 : LZ4HC_CLEVEL_MAX; - compressionLevel = MIN(compressionLevel, minCLevel); - compressionLevel = MAX(compressionLevel, maxCLevel); + if (compressionLevel < 1) compressionLevel = 1; + if (compressionLevel > LZ4HC_CLEVEL_MAX) compressionLevel = LZ4HC_CLEVEL_MAX; LZ4_streamHCPtr->internal_donotuse.compressionLevel = compressionLevel; } @@ -648,10 +742,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int } LZ4HC_init (ctxPtr, (const BYTE*)dictionary); ctxPtr->end = (const BYTE*)dictionary + dictSize; - if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN) - LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS); - else - if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); + if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); return dictSize; } @@ -660,10 +751,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) { - if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN) - LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS); - else - if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ + if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ ctxPtr->lowLimit = ctxPtr->dictLimit; @@ -717,8 +805,6 @@ int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int* srcSizePtr, int targetDestSize) { - LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; - if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN) LZ4HC_init(ctxPtr, (const BYTE*)src); /* not compatible with btopt implementation */ return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, dst, srcSizePtr, targetDestSize, limitedDestSize); } diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 66d5636..d41bf42 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -39,14 +39,14 @@ extern "C" { #endif /* --- Dependency --- */ -/* note : lz4hc is not an independent module, it requires lz4.h/lz4.c for proper compilation */ +/* note : lz4hc requires lz4.h/lz4.c for compilation */ #include "lz4.h" /* stddef, LZ4LIB_API, LZ4_DEPRECATED */ /* --- Useful constants --- */ #define LZ4HC_CLEVEL_MIN 3 #define LZ4HC_CLEVEL_DEFAULT 9 -#define LZ4HC_CLEVEL_OPT_MIN 11 +#define LZ4HC_CLEVEL_OPT_MIN 10 #define LZ4HC_CLEVEL_MAX 12 @@ -54,12 +54,12 @@ extern "C" { * Block Compression **************************************/ /*! LZ4_compress_HC() : - * Compress data from `src` into `dst`, using the more powerful but slower "HC" algorithm. + * Compress data from `src` into `dst`, using the more powerful but slower "HC" algorithm. * `dst` must be already allocated. - * Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h") - * Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h") - * `compressionLevel` : Recommended values are between 4 and 9, although any value between 1 and LZ4HC_CLEVEL_MAX will work. - * Values >LZ4HC_CLEVEL_MAX behave the same as LZ4HC_CLEVEL_MAX. + * Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h") + * Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h") + * `compressionLevel` : any value between 1 and LZ4HC_CLEVEL_MAX will work. + * Values > LZ4HC_CLEVEL_MAX behave the same as LZ4HC_CLEVEL_MAX. * @return : the number of bytes written into 'dst' * or 0 if compression fails. */ @@ -72,12 +72,12 @@ LZ4LIB_API int LZ4_compress_HC (const char* src, char* dst, int srcSize, int dst /*! LZ4_compress_HC_extStateHC() : - * Same as LZ4_compress_HC(), but using an externally allocated memory segment for `state`. + * Same as LZ4_compress_HC(), but using an externally allocated memory segment for `state`. * `state` size is provided by LZ4_sizeofStateHC(). - * Memory segment must be aligned on 8-bytes boundaries (which a normal malloc() will do properly). + * Memory segment must be aligned on 8-bytes boundaries (which a normal malloc() should do properly). */ -LZ4LIB_API int LZ4_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); LZ4LIB_API int LZ4_sizeofStateHC(void); +LZ4LIB_API int LZ4_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); /*-************************************ @@ -87,10 +87,10 @@ LZ4LIB_API int LZ4_sizeofStateHC(void); typedef union LZ4_streamHC_u LZ4_streamHC_t; /* incomplete type (defined later) */ /*! LZ4_createStreamHC() and LZ4_freeStreamHC() : - * These functions create and release memory for LZ4 HC streaming state. - * Newly created states are automatically initialized. - * Existing states can be re-used several times, using LZ4_resetStreamHC(). - * These methods are API and ABI stable, they can be used in combination with a DLL. + * These functions create and release memory for LZ4 HC streaming state. + * Newly created states are automatically initialized. + * Existing states can be re-used several times, using LZ4_resetStreamHC(). + * These methods are API and ABI stable, they can be used in combination with a DLL. */ LZ4LIB_API LZ4_streamHC_t* LZ4_createStreamHC(void); LZ4LIB_API int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr); @@ -123,13 +123,13 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in */ - /*-************************************* +/*-************************************************************** * PRIVATE DEFINITIONS : * Do not use these definitions. * They are exposed to allow static allocation of `LZ4_streamHC_t`. * Using these definitions makes the code vulnerable to potential API break when upgrading LZ4 - **************************************/ -#define LZ4HC_DICTIONARY_LOGSIZE 17 /* because of btopt, hc would only need 16 */ + ****************************************************************/ +#define LZ4HC_DICTIONARY_LOGSIZE 16 #define LZ4HC_MAXD (1<<LZ4HC_DICTIONARY_LOGSIZE) #define LZ4HC_MAXD_MASK (LZ4HC_MAXD - 1) @@ -152,8 +152,7 @@ typedef struct uint32_t dictLimit; /* below that point, need extDict */ uint32_t lowLimit; /* below that point, no more dict */ uint32_t nextToUpdate; /* index from which to continue dictionary update */ - uint32_t searchNum; /* only for optimal parser */ - uint32_t compressionLevel; + int compressionLevel; } LZ4HC_CCtx_internal; #else @@ -169,13 +168,12 @@ typedef struct unsigned int dictLimit; /* below that point, need extDict */ unsigned int lowLimit; /* below that point, no more dict */ unsigned int nextToUpdate; /* index from which to continue dictionary update */ - unsigned int searchNum; /* only for optimal parser */ int compressionLevel; } LZ4HC_CCtx_internal; #endif -#define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56) /* 393268 */ +#define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56) /* 262200 */ #define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t)) union LZ4_streamHC_u { size_t table[LZ4_STREAMHCSIZE_SIZET]; @@ -197,7 +195,6 @@ union LZ4_streamHC_u { /* see lz4.h LZ4_DISABLE_DEPRECATE_WARNINGS to turn off deprecation warnings */ /* deprecated compression functions */ -/* these functions will trigger warning messages in future releases */ LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC (const char* source, char* dest, int inputSize); LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel); @@ -225,14 +222,15 @@ LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStr #endif /* LZ4_HC_H_19834876238432 */ -/*-************************************************ + +/*-************************************************** * !!!!! STATIC LINKING ONLY !!!!! * Following definitions are considered experimental. * They should not be linked from DLL, * as there is no guarantee of API stability yet. * Prototypes will be promoted to "stable" status * after successfull usage in real-life scenarios. - *************************************************/ + ***************************************************/ #ifdef LZ4_HC_STATIC_LINKING_ONLY /* protection macro */ #ifndef LZ4_HC_SLO_098092834 #define LZ4_HC_SLO_098092834 @@ -258,17 +256,13 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, * @return : the number of bytes written into 'dst' * or 0 if compression fails. * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src`. - * Important : due to limitations, this prototype only works well up to cLevel < LZ4HC_CLEVEL_OPT_MIN - * beyond that level, compression performance will be much reduced due to internal incompatibilities */ int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int* srcSizePtr, int targetDstSize); /*! LZ4_setCompressionLevel() : v1.8.0 (experimental) - * It's possible to change compression level after LZ4_resetStreamHC(), between 2 invocations of LZ4_compress_HC_continue*(), - * but that requires to stay in the same mode (aka 1-10 or 11-12). - * This function ensures this condition. + * It's possible to change compression level between 2 invocations of LZ4_compress_HC_continue*() */ void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); diff --git a/lib/lz4opt.h b/lib/lz4opt.h index e9e54d8..5a8438c 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -35,12 +35,6 @@ #define LZ4_OPT_NUM (1<<12) - -typedef struct { - int off; - int len; -} LZ4HC_match_t; - typedef struct { int price; int off; @@ -50,317 +44,313 @@ typedef struct { /* price in bytes */ -FORCE_INLINE size_t LZ4HC_literalsPrice(size_t litlen) +LZ4_FORCE_INLINE int LZ4HC_literalsPrice(int const litlen) { - size_t price = litlen; - if (litlen >= (size_t)RUN_MASK) + int price = litlen; + if (litlen >= (int)RUN_MASK) price += 1 + (litlen-RUN_MASK)/255; return price; } /* requires mlen >= MINMATCH */ -FORCE_INLINE size_t LZ4HC_sequencePrice(size_t litlen, size_t mlen) +LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen) { - size_t price = 2 + 1; /* 16-bit offset + token */ + int price = 1 + 2 ; /* token + 16-bit offset */ price += LZ4HC_literalsPrice(litlen); - if (mlen >= (size_t)(ML_MASK+MINMATCH)) - price+= 1 + (mlen-(ML_MASK+MINMATCH))/255; + if (mlen >= (int)(ML_MASK+MINMATCH)) + price += 1 + (mlen-(ML_MASK+MINMATCH))/255; return price; } /*-************************************* -* Binary Tree search +* Match finder ***************************************/ -FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( - LZ4HC_CCtx_internal* ctx, - const BYTE* const ip, - const BYTE* const iHighLimit, - size_t best_mlen, - LZ4HC_match_t* matches, - int* matchNum) -{ - U16* const chainTable = ctx->chainTable; - U32* const HashTable = ctx->hashTable; - const BYTE* const base = ctx->base; - const U32 dictLimit = ctx->dictLimit; - const U32 current = (U32)(ip - base); - const U32 lowLimit = (ctx->lowLimit + MAX_DISTANCE > current) ? ctx->lowLimit : current - (MAX_DISTANCE - 1); - const BYTE* const dictBase = ctx->dictBase; - const BYTE* match; - int nbAttempts = ctx->searchNum; - int mnum = 0; - U16 *ptr0, *ptr1, delta0, delta1; - U32 matchIndex; - size_t matchLength = 0; - U32* HashPos; - - if (ip + MINMATCH > iHighLimit) return 1; - - /* HC4 match finder */ - HashPos = &HashTable[LZ4HC_hashPtr(ip)]; - matchIndex = *HashPos; - *HashPos = current; - - ptr0 = &DELTANEXTMAXD(current*2+1); - ptr1 = &DELTANEXTMAXD(current*2); - delta0 = delta1 = (U16)(current - matchIndex); - - while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) { - nbAttempts--; - if (matchIndex >= dictLimit) { - match = base + matchIndex; - matchLength = LZ4_count(ip, match, iHighLimit); - } else { - const BYTE* vLimit = ip + (dictLimit - matchIndex); - match = dictBase + matchIndex; - if (vLimit > iHighLimit) vLimit = iHighLimit; - matchLength = LZ4_count(ip, match, vLimit); - if ((ip+matchLength == vLimit) && (vLimit < iHighLimit)) - matchLength += LZ4_count(ip+matchLength, base+dictLimit, iHighLimit); - if (matchIndex+matchLength >= dictLimit) - match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ - } - - if (matchLength > best_mlen) { - best_mlen = matchLength; - if (matches) { - if (matchIndex >= dictLimit) - matches[mnum].off = (int)(ip - match); - else - matches[mnum].off = (int)(ip - (base + matchIndex)); /* virtual matchpos */ - matches[mnum].len = (int)matchLength; - mnum++; - } - if (best_mlen > LZ4_OPT_NUM) break; - } - - if (ip+matchLength >= iHighLimit) /* equal : no way to know if inf or sup */ - break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ - - DEBUGLOG(6, "ip :%016llX", (U64)ip); - DEBUGLOG(6, "match:%016llX", (U64)match); - if (*(ip+matchLength) < *(match+matchLength)) { - *ptr0 = delta0; - ptr0 = &DELTANEXTMAXD(matchIndex*2); - if (*ptr0 == (U16)-1) break; - delta0 = *ptr0; - delta1 += delta0; - matchIndex -= delta0; - } else { - *ptr1 = delta1; - ptr1 = &DELTANEXTMAXD(matchIndex*2+1); - if (*ptr1 == (U16)-1) break; - delta1 = *ptr1; - delta0 += delta1; - matchIndex -= delta1; - } - } - - *ptr0 = (U16)-1; - *ptr1 = (U16)-1; - if (matchNum) *matchNum = mnum; - /* if (best_mlen > 8) return best_mlen-8; */ - if (!matchNum) return 1; - return 1; -} - - -FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* const ip, const BYTE* const iHighLimit) -{ - const BYTE* const base = ctx->base; - const U32 target = (U32)(ip - base); - U32 idx = ctx->nextToUpdate; - while(idx < target) - idx += LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, 8, NULL, NULL); -} - +typedef struct { + int off; + int len; +} LZ4HC_match_t; -/** Tree updater, providing best match */ -FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( - LZ4HC_CCtx_internal* ctx, - const BYTE* const ip, const BYTE* const iHighLimit, - size_t best_mlen, LZ4HC_match_t* matches, const int fullUpdate) +LZ4_FORCE_INLINE +LZ4HC_match_t LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx, + const BYTE* ip, const BYTE* const iHighLimit, + int minLen, int nbSearches) { - int mnum = 0; - if (ip < ctx->base + ctx->nextToUpdate) return 0; /* skipped area */ - if (fullUpdate) LZ4HC_updateBinTree(ctx, ip, iHighLimit); - best_mlen = LZ4HC_BinTree_InsertAndGetAllMatches(ctx, ip, iHighLimit, best_mlen, matches, &mnum); - ctx->nextToUpdate = (U32)(ip - ctx->base + best_mlen); - return mnum; -} - - -#define SET_PRICE(pos, ml, offset, ll, cost) \ -{ \ - while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \ - opt[pos].mlen = (int)ml; \ - opt[pos].off = (int)offset; \ - opt[pos].litlen = (int)ll; \ - opt[pos].price = (int)cost; \ + LZ4HC_match_t match = { 0 , 0 }; + const BYTE* matchPtr = NULL; + /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos), + * but this won't be the case here, as we define iLowLimit==ip, + * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */ + int const matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, + ip, ip, iHighLimit, minLen, &matchPtr, &ip, + nbSearches, 1 /* patternAnalysis */); + if (matchLength <= minLen) return match; + match.len = matchLength; + match.off = (int)(ip-matchPtr); + return match; } static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, const char* const source, - char* dest, - int inputSize, - int maxOutputSize, - limitedOutput_directive limit, + char* dst, + int* srcSizePtr, + int dstCapacity, + int const nbSearches, size_t sufficient_len, - const int fullUpdate + limitedOutput_directive limit, + int const fullUpdate ) { - LZ4HC_optimal_t opt[LZ4_OPT_NUM + 1]; /* this uses a bit too much stack memory to my taste ... */ - LZ4HC_match_t matches[LZ4_OPT_NUM + 1]; +#define TRAILING_LITERALS 3 + LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS]; /* this uses a bit too much stack memory to my taste ... */ const BYTE* ip = (const BYTE*) source; const BYTE* anchor = ip; - const BYTE* const iend = ip + inputSize; + const BYTE* const iend = ip + *srcSizePtr; const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = (iend - LASTLITERALS); - BYTE* op = (BYTE*) dest; - BYTE* const oend = op + maxOutputSize; + const BYTE* const matchlimit = iend - LASTLITERALS; + BYTE* op = (BYTE*) dst; + BYTE* opSaved = (BYTE*) dst; + BYTE* oend = op + dstCapacity; /* init */ DEBUGLOG(5, "LZ4HC_compress_optimal"); + *srcSizePtr = 0; + if (limit == limitedDestSize) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1; - ctx->end += inputSize; - ip++; /* Main Loop */ + assert(ip - anchor < LZ4_MAX_INPUT_SIZE); while (ip < mflimit) { - size_t const llen = ip - anchor; - size_t last_pos = 0; - size_t match_num, cur, best_mlen, best_off; - memset(opt, 0, sizeof(LZ4HC_optimal_t)); /* memset only the first one */ + int const llen = (int)(ip - anchor); + int best_mlen, best_off; + int cur, last_match_pos = 0; - match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate); - if (!match_num) { ip++; continue; } + LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches); + if (firstMatch.len==0) { ip++; continue; } - if ((size_t)matches[match_num-1].len > sufficient_len) { + if ((size_t)firstMatch.len > sufficient_len) { /* good enough solution : immediate encoding */ - best_mlen = matches[match_num-1].len; - best_off = matches[match_num-1].off; - cur = 0; - last_pos = 1; - goto encode; + int const firstML = firstMatch.len; + const BYTE* const matchPos = ip - firstMatch.off; + opSaved = op; + if ( LZ4HC_encodeSequence(&ip, &op, &anchor, firstML, matchPos, limit, oend) ) /* updates ip, op and anchor */ + goto _dest_overflow; + continue; } - /* set prices using matches at position = 0 */ - { size_t matchNb; - for (matchNb = 0; matchNb < match_num; matchNb++) { - size_t mlen = (matchNb>0) ? (size_t)matches[matchNb-1].len+1 : MINMATCH; - best_mlen = matches[matchNb].len; /* necessarily < sufficient_len < LZ4_OPT_NUM */ - for ( ; mlen <= best_mlen ; mlen++) { - size_t const cost = LZ4HC_sequencePrice(llen, mlen) - LZ4HC_literalsPrice(llen); - SET_PRICE(mlen, mlen, matches[matchNb].off, 0, cost); /* updates last_pos and opt[pos] */ - } } } - - if (last_pos < MINMATCH) { ip++; continue; } /* note : on clang at least, this test improves performance */ + /* set prices for first positions (literals) */ + { int rPos; + for (rPos = 0 ; rPos < MINMATCH ; rPos++) { + int const cost = LZ4HC_literalsPrice(llen + rPos); + opt[rPos].mlen = 1; + opt[rPos].off = 0; + opt[rPos].litlen = llen + rPos; + opt[rPos].price = cost; + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup", + rPos, cost, opt[rPos].litlen); + } } + /* set prices using initial match */ + { int mlen = MINMATCH; + int const matchML = firstMatch.len; /* necessarily < sufficient_len < LZ4_OPT_NUM */ + int const offset = firstMatch.off; + assert(matchML < LZ4_OPT_NUM); + for ( ; mlen <= matchML ; mlen++) { + int const cost = LZ4HC_sequencePrice(llen, mlen); + opt[mlen].mlen = mlen; + opt[mlen].off = offset; + opt[mlen].litlen = llen; + opt[mlen].price = cost; + DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i) -- initial setup", + mlen, cost, mlen); + } } + last_match_pos = firstMatch.len; + { int addLit; + for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) { + opt[last_match_pos+addLit].mlen = 1; /* literal */ + opt[last_match_pos+addLit].off = 0; + opt[last_match_pos+addLit].litlen = addLit; + opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit); + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup", + last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit); + } } /* check further positions */ - opt[0].mlen = opt[1].mlen = 1; - for (cur = 1; cur <= last_pos; cur++) { + for (cur = 1; cur < last_match_pos; cur++) { const BYTE* const curPtr = ip + cur; - - /* establish baseline price if cur is literal */ - { size_t price, litlen; - if (opt[cur-1].mlen == 1) { - /* no match at previous position */ - litlen = opt[cur-1].litlen + 1; - if (cur > litlen) { - price = opt[cur - litlen].price + LZ4HC_literalsPrice(litlen); - } else { - price = LZ4HC_literalsPrice(llen + litlen) - LZ4HC_literalsPrice(llen); - } - } else { - litlen = 1; - price = opt[cur - 1].price + LZ4HC_literalsPrice(1); - } - - if (price < (size_t)opt[cur].price) - SET_PRICE(cur, 1 /*mlen*/, 0 /*off*/, litlen, price); /* note : increases last_pos */ + LZ4HC_match_t newMatch; + + if (curPtr >= mflimit) break; + DEBUGLOG(7, "rPos:%u[%u] vs [%u]%u", + cur, opt[cur].price, opt[cur+1].price, cur+1); + if (fullUpdate) { + /* not useful to search here if next position has same (or lower) cost */ + if ( (opt[cur+1].price <= opt[cur].price) + /* in some cases, next position has same cost, but cost rises sharply after, so a small match would still be beneficial */ + && (opt[cur+MINMATCH].price < opt[cur].price + 3/*min seq price*/) ) + continue; + } else { + /* not useful to search here if next position has same (or lower) cost */ + if (opt[cur+1].price <= opt[cur].price) continue; } - if (cur == last_pos || curPtr >= mflimit) break; + DEBUGLOG(7, "search at rPos:%u", cur); + if (fullUpdate) + newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches); + else + /* only test matches of minimum length; slightly faster, but misses a few bytes */ + newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches); + if (!newMatch.len) continue; - match_num = LZ4HC_BinTree_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate); - if ((match_num > 0) && (size_t)matches[match_num-1].len > sufficient_len) { + if ( ((size_t)newMatch.len > sufficient_len) + || (newMatch.len + cur >= LZ4_OPT_NUM) ) { /* immediate encoding */ - best_mlen = matches[match_num-1].len; - best_off = matches[match_num-1].off; - last_pos = cur + 1; + best_mlen = newMatch.len; + best_off = newMatch.off; + last_match_pos = cur + 1; goto encode; } - /* set prices using matches at position = cur */ - { size_t matchNb; - for (matchNb = 0; matchNb < match_num; matchNb++) { - size_t ml = (matchNb>0) ? (size_t)matches[matchNb-1].len+1 : MINMATCH; - best_mlen = (cur + matches[matchNb].len < LZ4_OPT_NUM) ? - (size_t)matches[matchNb].len : LZ4_OPT_NUM - cur; - - for ( ; ml <= best_mlen ; ml++) { - size_t ll, price; - if (opt[cur].mlen == 1) { - ll = opt[cur].litlen; - if (cur > ll) - price = opt[cur - ll].price + LZ4HC_sequencePrice(ll, ml); - else - price = LZ4HC_sequencePrice(llen + ll, ml) - LZ4HC_literalsPrice(llen); - } else { - ll = 0; - price = opt[cur].price + LZ4HC_sequencePrice(0, ml); - } - - if (cur + ml > last_pos || price < (size_t)opt[cur + ml].price) { - SET_PRICE(cur + ml, ml, matches[matchNb].off, ll, price); - } } } } - } /* for (cur = 1; cur <= last_pos; cur++) */ - - best_mlen = opt[last_pos].mlen; - best_off = opt[last_pos].off; - cur = last_pos - best_mlen; - -encode: /* cur, last_pos, best_mlen, best_off must be set */ - opt[0].mlen = 1; - while (1) { /* from end to beginning */ - size_t const ml = opt[cur].mlen; - int const offset = opt[cur].off; - opt[cur].mlen = (int)best_mlen; - opt[cur].off = (int)best_off; - best_mlen = ml; - best_off = offset; - if (ml > cur) break; /* can this happen ? */ - cur -= ml; - } + /* before match : set price with literals at beginning */ + { int const baseLitlen = opt[cur].litlen; + int litlen; + for (litlen = 1; litlen < MINMATCH; litlen++) { + int const price = opt[cur].price - LZ4HC_literalsPrice(baseLitlen) + LZ4HC_literalsPrice(baseLitlen+litlen); + int const pos = cur + litlen; + if (price < opt[pos].price) { + opt[pos].mlen = 1; /* literal */ + opt[pos].off = 0; + opt[pos].litlen = baseLitlen+litlen; + opt[pos].price = price; + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", + pos, price, opt[pos].litlen); + } } } + + /* set prices using match at position = cur */ + { int const matchML = newMatch.len; + int ml = MINMATCH; + + assert(cur + newMatch.len < LZ4_OPT_NUM); + for ( ; ml <= matchML ; ml++) { + int const pos = cur + ml; + int const offset = newMatch.off; + int price; + int ll; + DEBUGLOG(7, "testing price rPos %i (last_match_pos=%i)", + pos, last_match_pos); + if (opt[cur].mlen == 1) { + ll = opt[cur].litlen; + price = ((cur > ll) ? opt[cur - ll].price : 0) + + LZ4HC_sequencePrice(ll, ml); + } else { + ll = 0; + price = opt[cur].price + LZ4HC_sequencePrice(0, ml); + } - /* encode all recorded sequences */ - cur = 0; - while (cur < last_pos) { - int const ml = opt[cur].mlen; - int const offset = opt[cur].off; - if (ml == 1) { ip++; cur++; continue; } - cur += ml; - if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) ) return 0; - } + if (pos > last_match_pos+TRAILING_LITERALS || price <= opt[pos].price) { + DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i)", + pos, price, ml); + assert(pos < LZ4_OPT_NUM); + if ( (ml == matchML) /* last pos of last match */ + && (last_match_pos < pos) ) + last_match_pos = pos; + opt[pos].mlen = ml; + opt[pos].off = offset; + opt[pos].litlen = ll; + opt[pos].price = price; + } } } + /* complete following positions with literals */ + { int addLit; + for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) { + opt[last_match_pos+addLit].mlen = 1; /* literal */ + opt[last_match_pos+addLit].off = 0; + opt[last_match_pos+addLit].litlen = addLit; + opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit); + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit); + } } + } /* for (cur = 1; cur <= last_match_pos; cur++) */ + + best_mlen = opt[last_match_pos].mlen; + best_off = opt[last_match_pos].off; + cur = last_match_pos - best_mlen; + +encode: /* cur, last_match_pos, best_mlen, best_off must be set */ + assert(cur < LZ4_OPT_NUM); + assert(last_match_pos >= 1); /* == 1 when only one candidate */ + DEBUGLOG(6, "reverse traversal, looking for shortest path") + DEBUGLOG(6, "last_match_pos = %i", last_match_pos); + { int candidate_pos = cur; + int selected_matchLength = best_mlen; + int selected_offset = best_off; + while (1) { /* from end to beginning */ + int const next_matchLength = opt[candidate_pos].mlen; /* can be 1, means literal */ + int const next_offset = opt[candidate_pos].off; + DEBUGLOG(6, "pos %i: sequence length %i", candidate_pos, selected_matchLength); + opt[candidate_pos].mlen = selected_matchLength; + opt[candidate_pos].off = selected_offset; + selected_matchLength = next_matchLength; + selected_offset = next_offset; + if (next_matchLength > candidate_pos) break; /* last match elected, first match to encode */ + assert(next_matchLength > 0); /* can be 1, means literal */ + candidate_pos -= next_matchLength; + } } + + /* encode all recorded sequences in order */ + { int rPos = 0; /* relative position (to ip) */ + while (rPos < last_match_pos) { + int const ml = opt[rPos].mlen; + int const offset = opt[rPos].off; + if (ml == 1) { ip++; rPos++; continue; } /* literal; note: can end up with several literals, in which case, skip them */ + rPos += ml; + assert(ml >= MINMATCH); + assert((offset >= 1) && (offset <= MAX_DISTANCE)); + opSaved = op; + if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) ) /* updates ip, op and anchor */ + goto _dest_overflow; + } } } /* while (ip < mflimit) */ +_last_literals: /* Encode Last Literals */ - { int lastRun = (int)(iend - anchor); - if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ - if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun > 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } - else *op++ = (BYTE)(lastRun<<ML_BITS); - memcpy(op, anchor, iend - anchor); - op += iend-anchor; + { size_t lastRunSize = (size_t)(iend - anchor); /* literals */ + size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255; + size_t const totalSize = 1 + litLength + lastRunSize; + if (limit == limitedDestSize) oend += LASTLITERALS; /* restore correct value */ + if (limit && (op + totalSize > oend)) { + if (limit == limitedOutput) return 0; /* Check output limit */ + /* adapt lastRunSize to fill 'dst' */ + lastRunSize = (size_t)(oend - op) - 1; + litLength = (lastRunSize + 255 - RUN_MASK) / 255; + lastRunSize -= litLength; + } + ip = anchor + lastRunSize; + + if (lastRunSize >= RUN_MASK) { + size_t accumulator = lastRunSize - RUN_MASK; + *op++ = (RUN_MASK << ML_BITS); + for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRunSize << ML_BITS); + } + memcpy(op, anchor, lastRunSize); + op += lastRunSize; } /* End */ - return (int) ((char*)op-dest); + *srcSizePtr = (int) (((const char*)ip) - source); + return (int) ((char*)op-dst); + +_dest_overflow: + if (limit == limitedDestSize) { + op = opSaved; /* restore correct out pointer */ + goto _last_literals; + } + return 0; } diff --git a/lib/xxhash.c b/lib/xxhash.c index a532358..bcf1f1d 100644 --- a/lib/xxhash.c +++ b/lib/xxhash.c @@ -115,21 +115,21 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ #endif -#ifndef FORCE_INLINE +#ifndef XXH_FORCE_INLINE # ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline +# define XXH_FORCE_INLINE static __forceinline # else # if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ # ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) +# define XXH_FORCE_INLINE static inline __attribute__((always_inline)) # else -# define FORCE_INLINE static inline +# define XXH_FORCE_INLINE static inline # endif # else -# define FORCE_INLINE static +# define XXH_FORCE_INLINE static # endif /* __STDC_VERSION__ */ # endif /* _MSC_VER */ -#endif /* FORCE_INLINE */ +#endif /* XXH_FORCE_INLINE */ /* ************************************* @@ -223,7 +223,7 @@ typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; *****************************/ typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; -FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +XXH_FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) { if (align==XXH_unaligned) return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); @@ -231,7 +231,7 @@ FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_a return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); } -FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) +XXH_FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) { return XXH_readLE32_align(ptr, endian, XXH_unaligned); } @@ -266,7 +266,7 @@ static U32 XXH32_round(U32 seed, U32 input) return seed; } -FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) +XXH_FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) { const BYTE* p = (const BYTE*)input; const BYTE* bEnd = p + len; @@ -381,7 +381,7 @@ XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int s } -FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) +XXH_FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) { const BYTE* p = (const BYTE*)input; const BYTE* const bEnd = p + len; @@ -451,7 +451,7 @@ XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* -FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) +XXH_FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) { const BYTE * p = (const BYTE*)state->mem32; const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize; @@ -583,7 +583,7 @@ static U64 XXH_swap64 (U64 x) } #endif -FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +XXH_FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) { if (align==XXH_unaligned) return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); @@ -591,7 +591,7 @@ FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_a return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); } -FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) +XXH_FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) { return XXH_readLE64_align(ptr, endian, XXH_unaligned); } @@ -626,7 +626,7 @@ static U64 XXH64_mergeRound(U64 acc, U64 val) return acc; } -FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) +XXH_FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) { const BYTE* p = (const BYTE*)input; const BYTE* const bEnd = p + len; @@ -750,7 +750,7 @@ XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long return XXH_OK; } -FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) +XXH_FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) { const BYTE* p = (const BYTE*)input; const BYTE* const bEnd = p + len; @@ -815,7 +815,7 @@ XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* return XXH64_update_endian(state_in, input, len, XXH_bigEndian); } -FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) +XXH_FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) { const BYTE * p = (const BYTE*)state->mem64; const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize; diff --git a/programs/Makefile b/programs/Makefile index c7ef6d1..ed1a779 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -1,6 +1,6 @@ # ########################################################################## # LZ4 programs - Makefile -# Copyright (C) Yann Collet 2011-2016 +# Copyright (C) Yann Collet 2011-2017 # # This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets # @@ -41,14 +41,14 @@ LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) LIBVER := $(shell echo $(LIBVER_SCRIPT)) -SRCFILES := $(wildcard $(LZ4DIR)/*.c) $(wildcard *.c) -OBJFILES := $(patsubst %.c,%.o,$(SRCFILES)) +SRCFILES := $(sort $(wildcard $(LZ4DIR)/*.c) $(wildcard *.c)) +OBJFILES := $(SRCFILES:.c=.o) CPPFLAGS += -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_ CFLAGS ?= -O3 DEBUGFLAGS:=-Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \ - -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \ - -Wpointer-arith -Wstrict-aliasing=1 + -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \ + -Wpointer-arith -Wstrict-aliasing=1 CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) @@ -81,13 +81,13 @@ lz4: $(OBJFILES) lz4-release: DEBUGFLAGS= lz4-release: lz4 +lz4c: lz4 + ln -s lz4 lz4c + lz4c32: CFLAGS += -m32 lz4c32 : $(SRCFILES) $(CC) $(FLAGS) $^ -o $@$(EXT) -lz4c: lz4 - ln -s lz4 lz4c - lz4.1: lz4.1.md cat $^ | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@ @@ -117,14 +117,9 @@ unlz4: lz4 lz4cat: lz4 ln -s lz4 lz4cat -ifneq (,$(filter $(shell uname),SunOS)) -INSTALL ?= ginstall -else -INSTALL ?= install -endif - DESTDIR ?= -# directory variables : GNU convention prefers lowercase +# directory variables : GNU conventions prefer lowercase +# see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html # support both lower and uppercase (BSD), use uppercase in script prefix ?= /usr/local PREFIX ?= $(prefix) @@ -133,24 +128,31 @@ bindir ?= $(exec_prefix)/bin BINDIR ?= $(bindir) datarootdir ?= $(PREFIX)/share mandir ?= $(datarootdir)/man +man1dir ?= $(mandir)/man1 ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS)) MANDIR ?= $(PREFIX)/man/man1 else -MANDIR ?= $(mandir) +MANDIR ?= $(man1dir) +endif + +ifneq (,$(filter $(shell uname),SunOS)) +INSTALL ?= ginstall +else +INSTALL ?= install endif INSTALL_PROGRAM ?= $(INSTALL) -m 755 INSTALL_DATA ?= $(INSTALL) -m 644 -install: lz4$(EXT) lz4c$(EXT) +install: lz4 @echo Installing binaries @$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ @$(INSTALL_PROGRAM) lz4 $(DESTDIR)$(BINDIR)/lz4 + @ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4c @ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4cat @ln -sf lz4 $(DESTDIR)$(BINDIR)/unlz4 - @$(INSTALL_PROGRAM) lz4c$(EXT) $(DESTDIR)$(BINDIR)/lz4c @echo Installing man pages @$(INSTALL_DATA) lz4.1 $(DESTDIR)$(MANDIR)/lz4.1 @ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4c.1 diff --git a/programs/bench.c b/programs/bench.c index 77a9e3f..fac2a87 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -68,7 +68,7 @@ static int LZ4_compress_local(const char* src, char* dst, int srcSize, int dstSi #define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */ #define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */ #define COOLPERIOD_SEC 10 -#define DECOMP_MULT 2 /* test decompression DECOMP_MULT times longer than compression */ +#define DECOMP_MULT 1 /* test decompression DECOMP_MULT times longer than compression */ #define KB *(1 <<10) #define MB *(1 <<20) @@ -166,7 +166,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, void* const compressedBuffer = malloc(maxCompressedSize); void* const resultBuffer = malloc(srcSize); U32 nbBlocks; - UTIL_time_t ticksPerSecond; struct compressionParameters compP; int cfunctionId; @@ -176,7 +175,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, /* init */ if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */ - UTIL_initTimer(&ticksPerSecond); /* Init */ if (cLevel < LZ4HC_CLEVEL_MIN) cfunctionId = 0; else cfunctionId = 1; @@ -229,17 +227,17 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, size_t cSize = 0; double ratio = 0.; - UTIL_getTime(&coolTime); + coolTime = UTIL_getTime(); DISPLAYLEVEL(2, "\r%79s\r", ""); while (!cCompleted || !dCompleted) { UTIL_time_t clockStart; U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1; /* overheat protection */ - if (UTIL_clockSpanMicro(coolTime, ticksPerSecond) > ACTIVEPERIOD_MICROSEC) { + if (UTIL_clockSpanMicro(coolTime) > ACTIVEPERIOD_MICROSEC) { DISPLAYLEVEL(2, "\rcooling down ... \r"); UTIL_sleep(COOLPERIOD_SEC); - UTIL_getTime(&coolTime); + coolTime = UTIL_getTime(); } /* Compression */ @@ -247,8 +245,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */ UTIL_sleepMilli(1); /* give processor time to other processes */ - UTIL_waitForNextTick(ticksPerSecond); - UTIL_getTime(&clockStart); + UTIL_waitForNextTick(); + clockStart = UTIL_getTime(); if (!cCompleted) { /* still some time to do compression tests */ U32 nbLoops = 0; @@ -260,8 +258,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, blockTable[blockNb].cSize = rSize; } nbLoops++; - } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop); - { U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond); + } while (UTIL_clockSpanMicro(clockStart) < clockLoop); + { U64 const clockSpan = UTIL_clockSpanMicro(clockStart); if (clockSpan < fastestC*nbLoops) fastestC = clockSpan / nbLoops; totalCTime += clockSpan; cCompleted = totalCTime>maxTime; @@ -282,8 +280,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */ UTIL_sleepMilli(1); /* give processor time to other processes */ - UTIL_waitForNextTick(ticksPerSecond); - UTIL_getTime(&clockStart); + UTIL_waitForNextTick(); + clockStart = UTIL_getTime(); if (!dCompleted) { U32 nbLoops = 0; @@ -300,8 +298,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, blockTable[blockNb].resSize = regenSize; } nbLoops++; - } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < DECOMP_MULT*clockLoop); - { U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond); + } while (UTIL_clockSpanMicro(clockStart) < DECOMP_MULT*clockLoop); + { U64 const clockSpan = UTIL_clockSpanMicro(clockStart); if (clockSpan < fastestD*nbLoops) fastestD = clockSpan / nbLoops; totalDTime += clockSpan; dCompleted = totalDTime>(DECOMP_MULT*maxTime); @@ -428,7 +426,10 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize, f = fopen(fileNamesTable[n], "rb"); if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]); DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]); - if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ + if (fileSize > bufferSize-pos) { /* buffer too small - stop after this file */ + fileSize = bufferSize-pos; + nbFiles=n; + } { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]); pos += readSize; } @@ -456,9 +457,9 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, if (benchedSize==0) EXM_THROW(12, "not enough memory"); if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad; if (benchedSize > LZ4_MAX_INPUT_SIZE) { - benchedSize = LZ4_MAX_INPUT_SIZE; + benchedSize = LZ4_MAX_INPUT_SIZE; DISPLAY("File(s) bigger than LZ4's max input size; testing %u MB only...\n", (U32)(benchedSize >> 20)); - } else { + } else { if (benchedSize < totalSizeToLoad) DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20)); } diff --git a/programs/datagen.c b/programs/datagen.c index a61afc0..7285d69 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -119,7 +119,10 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match } /* init */ - if (pos==0) buffPtr[0] = RDG_genChar(seed, lt), pos=1; + if (pos==0) { + buffPtr[0] = RDG_genChar(seed, lt); + pos=1; + } /* Generate compressible data */ while (pos < buffSize) diff --git a/programs/lz4.1 b/programs/lz4.1 index 39d78cd..7ce5da0 100644 --- a/programs/lz4.1 +++ b/programs/lz4.1 @@ -35,7 +35,7 @@ When writing scripts that need to decompress files, it is recommended to always \fBlz4 file\.lz4\fR will default to decompression (use \fB\-z\fR to force compression) . .IP "\(bu" 4 -\fBlz4\fR shows real\-time notification statistics during compression or decompression of a single file (use \fB\-q\fR to silent them) +\fBlz4\fR shows real\-time notification statistics during compression or decompression of a single file (use \fB\-q\fR to silence them) . .IP "\(bu" 4 If no destination name is provided, result is sent to \fBstdout\fR \fIexcept if stdout is the console\fR\. diff --git a/programs/lz4.1.md b/programs/lz4.1.md index c6b99bc..b7b8570 100644 --- a/programs/lz4.1.md +++ b/programs/lz4.1.md @@ -37,7 +37,7 @@ Differences are : * `lz4 file.lz4` will default to decompression (use `-z` to force compression) * `lz4` shows real-time notification statistics during compression or decompression of a single file - (use `-q` to silent them) + (use `-q` to silence them) * If no destination name is provided, result is sent to `stdout` _except if stdout is the console_. * If no destination name is provided, __and__ if `stdout` is the console, @@ -45,6 +45,9 @@ Differences are : * As a consequence of previous rules, note the following example : `lz4 file | consumer` sends compressed data to `consumer` through `stdout`, hence it does _not_ create `file.lz4`. + * Another consequence of those rules is that to run `lz4` under `nohup`, + you should provide a destination file: `nohup lz4 file file.lz4`, + because `nohup` writes the specified command's output to a file. Default behaviors can be modified by opt-in commands, detailed below. diff --git a/programs/lz4cli.c b/programs/lz4cli.c index b4a3c14..362ba49 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -90,9 +90,6 @@ static unsigned displayLevel = 2; /* 0 : no display ; 1: errors only ; 2 : dow /*-************************************ * Version modifiers ***************************************/ -#define EXTENDED_ARGUMENTS -#define EXTENDED_HELP -#define EXTENDED_FORMAT #define DEFAULT_COMPRESSOR LZ4IO_compressFilename #define DEFAULT_DECOMPRESSOR LZ4IO_decompressFilename int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel); /* hidden function */ @@ -113,6 +110,7 @@ static int usage(const char* exeName) DISPLAY( " -9 : High compression \n"); DISPLAY( " -d : decompression (default for %s extension)\n", LZ4_EXTENSION); DISPLAY( " -z : force compression \n"); + DISPLAY( " -D FILE: use dictionary in FILE \n"); DISPLAY( " -f : overwrite output without prompting \n"); DISPLAY( " -k : preserve source files(s) (default) \n"); DISPLAY( "--rm : remove source file(s) after successful de/compression \n"); @@ -152,10 +150,9 @@ static int usage_advanced(const char* exeName) DISPLAY( "Legacy arguments : \n"); DISPLAY( " -c0 : fast compression \n"); DISPLAY( " -c1 : high compression \n"); - DISPLAY( " -hc : high compression \n"); + DISPLAY( " -c2,-hc: very high compression \n"); DISPLAY( " -y : overwrite output without prompting \n"); } - EXTENDED_HELP; return 0; } @@ -208,14 +205,14 @@ static int usage_longhelp(const char* exeName) DISPLAY( " generator | %s | consumer \n", exeName); if (g_lz4c_legacy_commands) { DISPLAY( "\n"); - DISPLAY( "***** Warning *****\n"); + DISPLAY( "***** Warning ***** \n"); DISPLAY( "Legacy arguments take precedence. Therefore : \n"); - DISPLAY( "---------------------------------\n"); - DISPLAY( " %s -hc filename\n", exeName); - DISPLAY( "means 'compress filename in high compression mode'\n"); - DISPLAY( "It is not equivalent to :\n"); - DISPLAY( " %s -h -c filename\n", exeName); - DISPLAY( "which would display help text and exit\n"); + DISPLAY( "--------------------------------- \n"); + DISPLAY( " %s -hc filename \n", exeName); + DISPLAY( "means 'compress filename in high compression mode' \n"); + DISPLAY( "It is not equivalent to : \n"); + DISPLAY( " %s -h -c filename \n", exeName); + DISPLAY( "which displays help text and exits \n"); } return 0; } @@ -259,8 +256,11 @@ static int exeNameMatch(const char* exeName, const char* test) static unsigned readU32FromChar(const char** stringPtr) { unsigned result = 0; - while ((**stringPtr >='0') && (**stringPtr <='9')) - result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + while ((**stringPtr >='0') && (**stringPtr <='9')) { + result *= 10; + result += **stringPtr - '0'; + (*stringPtr)++ ; + } if ((**stringPtr=='K') || (**stringPtr=='M')) { result <<= 10; if (**stringPtr=='M') result <<= 10; @@ -287,6 +287,7 @@ int main(int argc, const char** argv) operationMode_e mode = om_auto; const char* input_filename = NULL; const char* output_filename= NULL; + const char* dictionary_filename = NULL; char* dynNameSpace = NULL; const char** inFileNames = (const char**) calloc(argc, sizeof(char*)); unsigned ifnIdx=0; @@ -367,10 +368,11 @@ int main(int argc, const char** argv) if (g_lz4c_legacy_commands) { /* Legacy commands (-c0, -c1, -hc, -y) */ - if ((argument[0]=='c') && (argument[1]=='0')) { cLevel=0; argument++; continue; } /* -c0 (fast compression) */ - if ((argument[0]=='c') && (argument[1]=='1')) { cLevel=9; argument++; continue; } /* -c1 (high compression) */ - if ((argument[0]=='h') && (argument[1]=='c')) { cLevel=9; argument++; continue; } /* -hc (high compression) */ - if (argument[0]=='y') { LZ4IO_setOverwrite(1); continue; } /* -y (answer 'yes' to overwrite permission) */ + if (!strcmp(argument, "c0")) { cLevel=0; argument++; continue; } /* -c0 (fast compression) */ + if (!strcmp(argument, "c1")) { cLevel=9; argument++; continue; } /* -c1 (high compression) */ + if (!strcmp(argument, "c2")) { cLevel=12; argument++; continue; } /* -c2 (very high compression) */ + if (!strcmp(argument, "hc")) { cLevel=12; argument++; continue; } /* -hc (very high compression) */ + if (!strcmp(argument, "y")) { LZ4IO_setOverwrite(1); continue; } /* -y (answer 'yes' to overwrite permission) */ } if ((*argument>='0') && (*argument<='9')) { @@ -396,6 +398,22 @@ int main(int argc, const char** argv) /* Compression (default) */ case 'z': mode = om_compress; break; + case 'D': + if (argument[1] == '\0') { + /* path is next arg */ + if (i + 1 == argc) { + /* there is no next arg */ + badusage(exeName); + } + dictionary_filename = argv[++i]; + } else { + /* path follows immediately */ + dictionary_filename = argument + 1; + } + /* skip to end of argument so that we jump to parsing next argument */ + argument += strlen(argument) - 1; + break; + /* Use Legacy format (ex : Linux kernel compression) */ case 'l': legacy_format = 1; blockSize = 8 MB; break; @@ -485,9 +503,6 @@ int main(int argc, const char** argv) /* Pause at the end (hidden option) */ case 'p': main_pause=1; break; - /* Specific commands for customized versions */ - EXTENDED_ARGUMENTS; - /* Unrecognised command */ default : badusage(exeName); } @@ -539,8 +554,7 @@ int main(int argc, const char** argv) free((void*)inFileNames); inFileNames = extendedFileList; ifnIdx = fileNamesNb; - } - } + } } #endif } @@ -557,6 +571,14 @@ int main(int argc, const char** argv) mode = om_decompress; /* defer to decompress */ } + if (dictionary_filename) { + if (!strcmp(dictionary_filename, stdinmark) && IS_CONSOLE(stdin)) { + DISPLAYLEVEL(1, "refusing to read from a console\n"); + exit(1); + } + LZ4IO_setDictionaryFilename(dictionary_filename); + } + /* compress or decompress */ if (!input_filename) input_filename = stdinmark; /* Check if input is defined as console; trigger an error in this case */ @@ -624,7 +646,7 @@ int main(int argc, const char** argv) operationResult = DEFAULT_DECOMPRESSOR(input_filename, output_filename); } else { /* compression is default action */ if (legacy_format) { - DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated) ! \n"); + DISPLAYLEVEL(3, "! Generating LZ4 Legacy format (deprecated) ! \n"); LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel); } else { if (multiple_inputs) @@ -636,12 +658,13 @@ int main(int argc, const char** argv) _cleanup: if (main_pause) waitEnter(); - if (dynNameSpace) free(dynNameSpace); + free(dynNameSpace); #ifdef UTIL_HAS_CREATEFILELIST - if (extendedFileList) + if (extendedFileList) { UTIL_freeFileList(extendedFileList, fileNamesBuf); - else + inFileNames = NULL; + } #endif - free((void*)inFileNames); + free((void*)inFileNames); return operationResult; } diff --git a/programs/lz4io.c b/programs/lz4io.c index 06741b4..2cf0c1c 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -57,6 +57,7 @@ #include "lz4.h" /* still required for legacy format */ #include "lz4hc.h" /* still required for legacy format */ #include "lz4frame.h" +#include "lz4frame_static.h" /***************************** @@ -82,6 +83,7 @@ #define LEGACY_BLOCKSIZE (8 MB) #define MIN_STREAM_BUFSIZE (192 KB) #define LZ4IO_BLOCKSIZEID_DEFAULT 7 +#define LZ4_MAX_DICT_SIZE (64 KB) /************************************** @@ -110,6 +112,8 @@ static int g_streamChecksum = 1; static int g_blockIndependence = 1; static int g_sparseFileSupport = 1; static int g_contentSizeFlag = 0; +static int g_useDictionary = 0; +static const char* g_dictionaryFilename = NULL; /************************************** @@ -142,6 +146,12 @@ static int g_contentSizeFlag = 0; /* ****************** Parameters ******************** */ /* ************************************************** */ +int LZ4IO_setDictionaryFilename(const char* dictionaryFilename) { + g_dictionaryFilename = dictionaryFilename; + g_useDictionary = dictionaryFilename != NULL; + return g_useDictionary; +} + /* Default setting : overwrite = 1; return : overwrite mode (0/1) */ int LZ4IO_setOverwrite(int yes) { @@ -395,8 +405,79 @@ typedef struct { void* dstBuffer; size_t dstBufferSize; LZ4F_compressionContext_t ctx; + LZ4F_CDict* cdict; } cRess_t; +static void* LZ4IO_createDict(const char* dictFilename, size_t *dictSize) { + size_t readSize; + size_t dictEnd = 0; + size_t dictLen = 0; + size_t dictStart; + size_t circularBufSize = LZ4_MAX_DICT_SIZE; + char* circularBuf; + char* dictBuf; + FILE* dictFile; + + if (!dictFilename) EXM_THROW(25, "Dictionary error : no filename provided"); + + circularBuf = (char *) malloc(circularBufSize); + if (!circularBuf) EXM_THROW(25, "Allocation error : not enough memory"); + + dictFile = LZ4IO_openSrcFile(dictFilename); + if (!dictFile) EXM_THROW(25, "Dictionary error : could not open dictionary file"); + + /* opportunistically seek to the part of the file we care about. If this */ + /* fails it's not a problem since we'll just read everything anyways. */ + if (strcmp(dictFilename, stdinmark)) { + UTIL_fseek(dictFile, -LZ4_MAX_DICT_SIZE, SEEK_END); + } + + do { + readSize = fread(circularBuf + dictEnd, 1, circularBufSize - dictEnd, dictFile); + dictEnd = (dictEnd + readSize) % circularBufSize; + dictLen += readSize; + } while (readSize>0); + + if (dictLen > LZ4_MAX_DICT_SIZE) { + dictLen = LZ4_MAX_DICT_SIZE; + } + + *dictSize = dictLen; + + dictStart = (circularBufSize + dictEnd - dictLen) % circularBufSize; + + if (dictStart == 0) { + /* We're in the simple case where the dict starts at the beginning of our circular buffer. */ + dictBuf = circularBuf; + circularBuf = NULL; + } else { + /* Otherwise, we will alloc a new buffer and copy our dict into that. */ + dictBuf = (char *) malloc(dictLen ? dictLen : 1); + if (!dictBuf) EXM_THROW(25, "Allocation error : not enough memory"); + + memcpy(dictBuf, circularBuf + dictStart, circularBufSize - dictStart); + memcpy(dictBuf + circularBufSize - dictStart, circularBuf, dictLen - (circularBufSize - dictStart)); + } + + free(circularBuf); + + return dictBuf; +} + +static LZ4F_CDict* LZ4IO_createCDict(void) { + size_t dictionarySize; + void* dictionaryBuffer; + LZ4F_CDict* cdict; + if (!g_useDictionary) { + return NULL; + } + dictionaryBuffer = LZ4IO_createDict(g_dictionaryFilename, &dictionarySize); + if (!dictionaryBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary"); + cdict = LZ4F_createCDict(dictionaryBuffer, dictionarySize); + free(dictionaryBuffer); + return cdict; +} + static cRess_t LZ4IO_createCResources(void) { const size_t blockSize = (size_t)LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId); @@ -412,6 +493,8 @@ static cRess_t LZ4IO_createCResources(void) ress.dstBuffer = malloc(ress.dstBufferSize); if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory"); + ress.cdict = LZ4IO_createCDict(); + return ress; } @@ -419,6 +502,10 @@ static void LZ4IO_freeCResources(cRess_t ress) { free(ress.srcBuffer); free(ress.dstBuffer); + + LZ4F_freeCDict(ress.cdict); + ress.cdict = NULL; + { LZ4F_errorCode_t const errorCode = LZ4F_freeCompressionContext(ress.ctx); if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); } } @@ -472,7 +559,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, /* single-block file */ if (readSize < blockSize) { /* Compress in single pass */ - size_t const cSize = LZ4F_compressFrame(dstBuffer, dstBufferSize, srcBuffer, readSize, &prefs); + size_t cSize = LZ4F_compressFrame_usingCDict(dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs); if (LZ4F_isError(cSize)) EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize)); compressedfilesize = cSize; DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", @@ -488,7 +575,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, /* multiple-blocks file */ { /* Write Archive Header */ - size_t headerSize = LZ4F_compressBegin(ctx, dstBuffer, dstBufferSize, &prefs); + size_t headerSize = LZ4F_compressBegin_usingCDict(ctx, dstBuffer, dstBufferSize, ress.cdict, &prefs); if (LZ4F_isError(headerSize)) EXM_THROW(33, "File header generation failed : %s", LZ4F_getErrorName(headerSize)); { size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); if (sizeCheck!=headerSize) EXM_THROW(34, "Write error : cannot write header"); } @@ -529,7 +616,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, /* Copy owner, file permissions and modification time */ { stat_t statbuf; - if (strcmp (srcFileName, stdinmark) && strcmp (dstFileName, stdoutmark) && UTIL_getFileStat(srcFileName, &statbuf)) + if (strcmp (srcFileName, stdinmark) && strcmp (dstFileName, stdoutmark) && strcmp (dstFileName, nulmark) && UTIL_getFileStat(srcFileName, &statbuf)) UTIL_setFileStat(dstFileName, &statbuf); } @@ -745,8 +832,21 @@ typedef struct { size_t dstBufferSize; FILE* dstFile; LZ4F_decompressionContext_t dCtx; + void* dictBuffer; + size_t dictBufferSize; } dRess_t; +static void LZ4IO_loadDDict(dRess_t* ress) { + if (!g_useDictionary) { + ress->dictBuffer = NULL; + ress->dictBufferSize = 0; + return; + } + + ress->dictBuffer = LZ4IO_createDict(g_dictionaryFilename, &ress->dictBufferSize); + if (!ress->dictBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary"); +} + static const size_t LZ4IO_dBufferSize = 64 KB; static dRess_t LZ4IO_createDResources(void) { @@ -763,6 +863,8 @@ static dRess_t LZ4IO_createDResources(void) ress.dstBuffer = malloc(ress.dstBufferSize); if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory"); + LZ4IO_loadDDict(&ress); + ress.dstFile = NULL; return ress; } @@ -773,6 +875,7 @@ static void LZ4IO_freeDResources(dRess_t ress) if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); free(ress.srcBuffer); free(ress.dstBuffer); + free(ress.dictBuffer); } @@ -786,7 +889,7 @@ static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE { size_t inSize = MAGICNUMBER_SIZE; size_t outSize= 0; LZ4IO_writeLE32(ress.srcBuffer, LZ4IO_MAGICNUMBER); - nextToLoad = LZ4F_decompress(ress.dCtx, ress.dstBuffer, &outSize, ress.srcBuffer, &inSize, NULL); + nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, ress.dstBuffer, &outSize, ress.srcBuffer, &inSize, ress.dictBuffer, ress.dictBufferSize, NULL); if (LZ4F_isError(nextToLoad)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(nextToLoad)); } @@ -805,7 +908,7 @@ static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE /* Decode Input (at least partially) */ size_t remaining = readSize - pos; decodedBytes = ress.dstBufferSize; - nextToLoad = LZ4F_decompress(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, NULL); + nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, ress.dictBuffer, ress.dictBufferSize, NULL); if (LZ4F_isError(nextToLoad)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad)); pos += remaining; @@ -970,23 +1073,27 @@ static int LZ4IO_decompressSrcFile(dRess_t ress, const char* input_filename, con static int LZ4IO_decompressDstFile(dRess_t ress, const char* input_filename, const char* output_filename) { + stat_t statbuf; + int stat_result = 0; FILE* const foutput = LZ4IO_openDstFile(output_filename); if (foutput==NULL) return 1; /* failure */ + if ( strcmp(input_filename, stdinmark) + && UTIL_getFileStat(input_filename, &statbuf)) + stat_result = 1; + ress.dstFile = foutput; LZ4IO_decompressSrcFile(ress, input_filename, output_filename); fclose(foutput); /* Copy owner, file permissions and modification time */ - { stat_t statbuf; - if ( strcmp (input_filename, stdinmark) - && strcmp (output_filename, stdoutmark) - && strcmp (output_filename, nulmark) - && UTIL_getFileStat(input_filename, &statbuf) ) { - UTIL_setFileStat(output_filename, &statbuf); - /* should return value be read ? or is silent fail good enough ? */ - } } + if ( stat_result != 0 + && strcmp (output_filename, stdoutmark) + && strcmp (output_filename, nulmark)) { + UTIL_setFileStat(output_filename, &statbuf); + /* should return value be read ? or is silent fail good enough ? */ + } return 0; } diff --git a/programs/lz4io.h b/programs/lz4io.h index 6190f00..b21b8b6 100644 --- a/programs/lz4io.h +++ b/programs/lz4io.h @@ -64,6 +64,8 @@ int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSiz /* ****************** Parameters ******************** */ /* ************************************************** */ +int LZ4IO_setDictionaryFilename(const char* dictionaryFilename); + /* Default setting : overwrite = 1; return : overwrite mode (0/1) */ int LZ4IO_setOverwrite(int yes); diff --git a/programs/platform.h b/programs/platform.h index 66491b6..db2efac 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -30,10 +30,10 @@ extern "C" { * Compiler Options ****************************************/ #if defined(_MSC_VER) -# define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */ -# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before <io.h> and <windows.h> */ -# if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */ -# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ +# define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */ +# if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */ +# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before <io.h> and <windows.h> */ +# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ # endif #endif @@ -60,7 +60,7 @@ extern "C" { * Turn on Large Files support (>4GB) for 32-bit Linux/Unix ***********************************************************/ #if !defined(__64BIT__) || defined(__MINGW32__) /* No point defining Large file for 64 bit but MinGW-w64 requires it */ -# if !defined(_FILE_OFFSET_BITS) +# if !defined(_FILE_OFFSET_BITS) # define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ # endif # if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */ diff --git a/programs/util.h b/programs/util.h index 5a69c55..fc7f63e 100644 --- a/programs/util.h +++ b/programs/util.h @@ -140,45 +140,116 @@ extern "C" { /*-**************************************** * Time functions ******************************************/ -#if (PLATFORM_POSIX_VERSION >= 1) -#include <unistd.h> -#include <sys/times.h> /* times */ - typedef U64 UTIL_time_t; - UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { *ticksPerSecond=sysconf(_SC_CLK_TCK); } - UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { struct tms junk; clock_t newTicks = (clock_t) times(&junk); (void)junk; *x = (UTIL_time_t)newTicks; } - UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / ticksPerSecond; } - UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / ticksPerSecond; } -#elif defined(_WIN32) /* Windows */ - typedef LARGE_INTEGER UTIL_time_t; - UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { if (!QueryPerformanceFrequency(ticksPerSecond)) fprintf(stderr, "ERROR: QueryPerformance not present\n"); } - UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { QueryPerformanceCounter(x); } - UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; } - UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; } +#if defined(_WIN32) /* Windows */ + typedef LARGE_INTEGER UTIL_time_t; + UTIL_STATIC UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; } + UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) + { + static LARGE_INTEGER ticksPerSecond; + static int init = 0; + if (!init) { + if (!QueryPerformanceFrequency(&ticksPerSecond)) + fprintf(stderr, "ERROR: QueryPerformanceFrequency() failure\n"); + init = 1; + } + return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; + } + UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) + { + static LARGE_INTEGER ticksPerSecond; + static int init = 0; + if (!init) { + if (!QueryPerformanceFrequency(&ticksPerSecond)) + fprintf(stderr, "ERROR: QueryPerformanceFrequency() failure\n"); + init = 1; + } + return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; + } +#elif defined(__APPLE__) && defined(__MACH__) + #include <mach/mach_time.h> + typedef U64 UTIL_time_t; + UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); } + UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) + { + static mach_timebase_info_data_t rate; + static int init = 0; + if (!init) { + mach_timebase_info(&rate); + init = 1; + } + return (((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom))/1000ULL; + } + UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) + { + static mach_timebase_info_data_t rate; + static int init = 0; + if (!init) { + mach_timebase_info(&rate); + init = 1; + } + return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom); + } +#elif (PLATFORM_POSIX_VERSION >= 200112L) + #include <time.h> + typedef struct timespec UTIL_time_t; + UTIL_STATIC UTIL_time_t UTIL_getTime(void) + { + UTIL_time_t now; + if (clock_gettime(CLOCK_MONOTONIC, &now)) + fprintf(stderr, "ERROR: Failed to get time\n"); /* we could also exit() */ + return now; + } + UTIL_STATIC UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end) + { + UTIL_time_t diff; + if (end.tv_nsec < begin.tv_nsec) { + diff.tv_sec = (end.tv_sec - 1) - begin.tv_sec; + diff.tv_nsec = (end.tv_nsec + 1000000000ULL) - begin.tv_nsec; + } else { + diff.tv_sec = end.tv_sec - begin.tv_sec; + diff.tv_nsec = end.tv_nsec - begin.tv_nsec; + } + return diff; + } + UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end) + { + UTIL_time_t const diff = UTIL_getSpanTime(begin, end); + U64 micro = 0; + micro += 1000000ULL * diff.tv_sec; + micro += diff.tv_nsec / 1000ULL; + return micro; + } + UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end) + { + UTIL_time_t const diff = UTIL_getSpanTime(begin, end); + U64 nano = 0; + nano += 1000000000ULL * diff.tv_sec; + nano += diff.tv_nsec; + return nano; + } #else /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */ - typedef clock_t UTIL_time_t; - UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { *ticksPerSecond=0; } - UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { *x = clock(); } - UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; } - UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; } + typedef clock_t UTIL_time_t; + UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return clock(); } + UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; } + UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; } #endif /* returns time span in microseconds */ -UTIL_STATIC U64 UTIL_clockSpanMicro( UTIL_time_t clockStart, UTIL_time_t ticksPerSecond ) +UTIL_STATIC U64 UTIL_clockSpanMicro(UTIL_time_t clockStart) { - UTIL_time_t clockEnd; - UTIL_getTime(&clockEnd); - return UTIL_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd); + UTIL_time_t const clockEnd = UTIL_getTime(); + return UTIL_getSpanTimeMicro(clockStart, clockEnd); } -UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond) +UTIL_STATIC void UTIL_waitForNextTick(void) { - UTIL_time_t clockStart, clockEnd; - UTIL_getTime(&clockStart); + UTIL_time_t const clockStart = UTIL_getTime(); + UTIL_time_t clockEnd; do { - UTIL_getTime(&clockEnd); - } while (UTIL_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0); + clockEnd = UTIL_getTime(); + } while (UTIL_getSpanTimeNano(clockStart, clockEnd) == 0); } @@ -194,11 +265,17 @@ UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond) #endif +UTIL_STATIC int UTIL_isRegFile(const char* infilename); + + UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf) { int res = 0; struct utimbuf timebuf; + if (!UTIL_isRegFile(filename)) + return -1; + timebuf.actime = time(NULL); timebuf.modtime = statbuf->st_mtime; res += utime(filename, &timebuf); /* set access and modification times */ diff --git a/tests/Makefile b/tests/Makefile index f00778f..819ba43 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,6 +1,6 @@ # ########################################################################## # LZ4 programs - Makefile -# Copyright (C) Yann Collet 2011-2016 +# Copyright (C) Yann Collet 2011-2017 # # This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets # @@ -30,30 +30,28 @@ # datagen : generates synthetic data samples for tests & benchmarks # ########################################################################## -DESTDIR ?= -PREFIX ?= /usr/local -BINDIR := $(PREFIX)/bin -MANDIR := $(PREFIX)/share/man/man1 LZ4DIR := ../lib PRGDIR := ../programs -VOID := /dev/null TESTDIR := versionsTest PYTHON ?= python3 +DEBUGFLAGS = -g -DLZ4_DEBUG=1 CFLAGS ?= -O3 # can select custom optimization flags. For example : CFLAGS=-O2 make -CFLAGS += -g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ - -Wdeclaration-after-statement -Wstrict-prototypes \ +CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \ + -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \ -Wpointer-arith -Wstrict-aliasing=1 -CFLAGS += $(MOREFLAGS) +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) CPPFLAGS:= -I$(LZ4DIR) -I$(PRGDIR) -DXXH_NAMESPACE=LZ4_ FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) -EXT =.exe +EXT =.exe +VOID = nul else -EXT = +EXT = +VOID = /dev/null endif LZ4 := $(PRGDIR)/lz4$(EXT) @@ -66,39 +64,40 @@ NB_LOOPS ?= -i1 default: all -all: fullbench fuzzer frametest datagen fasttest +all: fullbench fuzzer frametest datagen all32: CFLAGS+=-m32 all32: all lz4: - $(MAKE) -C $(PRGDIR) clean $@ CFLAGS="$(CFLAGS)" + $(MAKE) -C $(PRGDIR) $@ CFLAGS="$(CFLAGS)" -lz4c: - $(MAKE) -C $(PRGDIR) clean $@ CFLAGS="$(CFLAGS)" +lz4c unlz4 lz4cat: lz4 + ln -sf $(LZ4) $(PRGDIR)/$@ lz4c32: # create a 32-bits version for 32/64 interop tests - $(MAKE) -C $(PRGDIR) clean $@ CFLAGS="-m32 $(CFLAGS)" - cp $(LZ4) $(LZ4)c32 + $(MAKE) -C $(PRGDIR) $@ CFLAGS="-m32 $(CFLAGS)" -fullbench : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o fullbench.c +%.o : $(LZ4DIR)/%.c $(LZ4DIR)/%.h + $(CC) -c $(CFLAGS) $(CPPFLAGS) $< -o $@ + +fullbench : lz4.o lz4hc.o lz4frame.o xxhash.o fullbench.c $(CC) $(FLAGS) $^ -o $@$(EXT) -fullbench-lib: fullbench.c $(LZ4DIR)/xxhash.c +$(LZ4DIR)/liblz4.a: $(MAKE) -C $(LZ4DIR) liblz4.a - $(CC) $(FLAGS) $^ -o $@$(EXT) $(LZ4DIR)/liblz4.a + +fullbench-lib: fullbench.c $(LZ4DIR)/liblz4.a + $(CC) $(FLAGS) $^ -o $@$(EXT) fullbench-dll: fullbench.c $(LZ4DIR)/xxhash.c $(MAKE) -C $(LZ4DIR) liblz4 $(CC) $(FLAGS) $^ -o $@$(EXT) -DLZ4_DLL_IMPORT=1 $(LZ4DIR)/dll/liblz4.dll -fuzzer : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/xxhash.o fuzzer.c - $(CC) $(FLAGS) $^ -o $@$(EXT) - -frametest: $(LZ4DIR)/lz4frame.o $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/xxhash.o frametest.c +fuzzer : lz4.o lz4hc.o xxhash.o fuzzer.c $(CC) $(FLAGS) $^ -o $@$(EXT) -fasttest: $(LZ4DIR)/lz4.o fasttest.c +frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c $(CC) $(FLAGS) $^ -o $@$(EXT) datagen : $(PRGDIR)/datagen.c datagencli.c @@ -122,7 +121,7 @@ versionsTest: #----------------------------------------------------------------------------- -# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets +# validated only for Linux, OSX, BSD, Hurd and Solaris targets #----------------------------------------------------------------------------- ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) @@ -136,160 +135,180 @@ ifneq (,$(filter $(shell uname),SunOS)) DIFF:=gdiff endif +DD:=dd -test: test-lz4 test-lz4c test-fasttest test-frametest test-fullbench test-fuzzer + +test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test32: CFLAGS+=-m32 test32: test test-lz4-sparse: lz4 datagen @echo "\n ---- test sparse file support ----" - ./datagen -g5M -P100 > tmpSrc - $(LZ4) -B4D tmpSrc | $(LZ4) -dv --sparse > tmpB4 - $(DIFF) -s tmpSrc tmpB4 - $(LZ4) -B5D tmpSrc | $(LZ4) -dv --sparse > tmpB5 - $(DIFF) -s tmpSrc tmpB5 - $(LZ4) -B6D tmpSrc | $(LZ4) -dv --sparse > tmpB6 - $(DIFF) -s tmpSrc tmpB6 - $(LZ4) -B7D tmpSrc | $(LZ4) -dv --sparse > tmpB7 - $(DIFF) -s tmpSrc tmpB7 - $(LZ4) tmpSrc | $(LZ4) -dv --no-sparse > tmpNoSparse - $(DIFF) -s tmpSrc tmpNoSparse - ls -ls tmp* - ./datagen -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > tmpOdd # Odd size file (to generate non-full last block) - ./datagen -s1 -g1200007 -P100 | $(DIFF) -s - tmpOdd - ls -ls tmpOdd - @$(RM) tmp* + ./datagen -g5M -P100 > tmplsdg5M + $(LZ4) -B4D tmplsdg5M | $(LZ4) -dv --sparse > tmplscB4 + $(DIFF) -s tmplsdg5M tmplscB4 + $(LZ4) -B5D tmplsdg5M | $(LZ4) -dv --sparse > tmplscB5 + $(DIFF) -s tmplsdg5M tmplscB5 + $(LZ4) -B6D tmplsdg5M | $(LZ4) -dv --sparse > tmplscB6 + $(DIFF) -s tmplsdg5M tmplscB6 + $(LZ4) -B7D tmplsdg5M | $(LZ4) -dv --sparse > tmplscB7 + $(DIFF) -s tmplsdg5M tmplscB7 + $(LZ4) tmplsdg5M | $(LZ4) -dv --no-sparse > tmplsnosparse + $(DIFF) -s tmplsdg5M tmplsnosparse + ls -ls tmpls* + ./datagen -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > tmplsodd # Odd size file (to generate non-full last block) + ./datagen -s1 -g1200007 -P100 | $(DIFF) -s - tmplsodd + ls -ls tmplsodd + @$(RM) tmpls* @echo "\n Compatibility with Console :" echo "Hello World 1 !" | $(LZ4) | $(LZ4) -d -c echo "Hello World 2 !" | $(LZ4) | $(LZ4) -d | cat echo "Hello World 3 !" | $(LZ4) --no-frame-crc | $(LZ4) -d -c @echo "\n Compatibility with Append :" - ./datagen -P100 -g1M > tmp1M - cat tmp1M tmp1M > tmp2M - $(LZ4) -B5 -v tmp1M tmpC - $(LZ4) -d -v tmpC tmpR - $(LZ4) -d -v tmpC >> tmpR + ./datagen -P100 -g1M > tmplsdg1M + cat tmplsdg1M tmplsdg1M > tmpls2M + $(LZ4) -B5 -v tmplsdg1M tmplsc + $(LZ4) -d -v tmplsc tmplsr + $(LZ4) -d -v tmplsc >> tmplsr ls -ls tmp* - $(DIFF) tmp2M tmpR - @$(RM) tmp* + $(DIFF) tmpls2M tmplsr + @$(RM) tmpls* test-lz4-contentSize: lz4 datagen @echo "\n ---- test original size support ----" - ./datagen -g15M > tmp - $(LZ4) -v tmp | $(LZ4) -t - $(LZ4) -v --content-size tmp | $(LZ4) -d > tmp2 - $(DIFF) -s tmp tmp2 + ./datagen -g15M > tmplc1 + $(LZ4) -v tmplc1 | $(LZ4) -t + $(LZ4) -v --content-size tmplc1 | $(LZ4) -d > tmplc2 + $(DIFF) -s tmplc1 tmplc2 # test large size [2-4] GB - @./datagen -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmp - @ls -ls tmp - @./datagen -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmp2 - @ls -ls tmp2 - $(DIFF) -s tmp tmp2 - @$(RM) tmp* + @./datagen -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmplc1 + @ls -ls tmplc1 + @./datagen -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmplc2 + @ls -ls tmplc2 + $(DIFF) -s tmplc1 tmplc2 + @$(RM) tmplc* test-lz4-frame-concatenation: lz4 datagen @echo "\n ---- test frame concatenation ----" - @echo -n > empty.test - @echo hi > nonempty.test - cat nonempty.test empty.test nonempty.test > orig.test - @$(LZ4) -zq empty.test > empty.lz4.test - @$(LZ4) -zq nonempty.test > nonempty.lz4.test - cat nonempty.lz4.test empty.lz4.test nonempty.lz4.test > concat.lz4.test - $(LZ4) -d concat.lz4.test > result.test - sdiff orig.test result.test - @$(RM) *.test + @echo -n > tmp-lfc-empty + @echo hi > tmp-lfc-nonempty + cat tmp-lfc-nonempty tmp-lfc-empty tmp-lfc-nonempty > tmp-lfc-src + @$(LZ4) -zq tmp-lfc-empty > tmp-lfc-empty.lz4 + @$(LZ4) -zq tmp-lfc-nonempty > tmp-lfc-nonempty.lz4 + cat tmp-lfc-nonempty.lz4 tmp-lfc-empty.lz4 tmp-lfc-nonempty.lz4 > tmp-lfc-concat.lz4 + $(LZ4) -d tmp-lfc-concat.lz4 > tmp-lfc-result + sdiff tmp-lfc-src tmp-lfc-result + @$(RM) tmp-lfc-* @echo frame concatenation test completed test-lz4-multiple: lz4 datagen @echo "\n ---- test multiple files ----" - @./datagen -s1 > tmp1 2> $(VOID) - @./datagen -s2 -g100K > tmp2 2> $(VOID) - @./datagen -s3 -g1M > tmp3 2> $(VOID) - $(LZ4) -f -m tmp* - ls -ls tmp* - @$(RM) tmp1 tmp2 tmp3 - $(LZ4) -df -m *.lz4 - ls -ls tmp* - $(LZ4) -f -m tmp1 notHere tmp2; echo $$? - @$(RM) tmp* - -unlz4: - @$(MAKE) -C $(PRGDIR) $@ CFLAGS="$(CFLAGS)" - -lz4cat: - @$(MAKE) -C $(PRGDIR) $@ CFLAGS="$(CFLAGS)" + @./datagen -s1 > tmp-tlm1 2> $(VOID) + @./datagen -s2 -g100K > tmp-tlm2 2> $(VOID) + @./datagen -s3 -g1M > tmp-tlm3 2> $(VOID) + $(LZ4) -f -m tmp-tlm* + ls -ls tmp-tlm* + @$(RM) tmp-tlm1 tmp-tlm2 tmp-tlm3 + $(LZ4) -df -m tmp-tlm*.lz4 + ls -ls tmp-tlm* + $(LZ4) -f -m tmp-tlm1 notHere tmp-tlm2; echo $$? + @$(RM) tmp-tlm* test-lz4-basic: lz4 datagen unlz4 lz4cat @echo "\n ---- test lz4 basic compression/decompression ----" - ./datagen -g0 | $(LZ4) -v | $(LZ4) -t - ./datagen -g16KB | $(LZ4) -9 | $(LZ4) -t - ./datagen -g20KB > tmpSrc - $(LZ4) < tmpSrc | $(LZ4) -d > tmpRes - $(DIFF) -q tmpSrc tmpRes - $(LZ4) --no-frame-crc < tmpSrc | $(LZ4) -d > tmpRes - $(DIFF) -q tmpSrc tmpRes - ./datagen | $(LZ4) | $(LZ4) -t - ./datagen -g6M -P99 | $(LZ4) -9BD | $(LZ4) -t - ./datagen -g17M | $(LZ4) -9v | $(LZ4) -qt - ./datagen -g33M | $(LZ4) --no-frame-crc | $(LZ4) -t - ./datagen -g256MB | $(LZ4) -vqB4D | $(LZ4) -t - @echo "hello world" > tmp - $(LZ4) --rm -f tmp tmp.lz4 - test ! -f tmp # must fail (--rm) - test -f tmp.lz4 - $(PRGDIR)/lz4cat tmp.lz4 # must display hello world - test -f tmp.lz4 - $(PRGDIR)/unlz4 --rm tmp.lz4 tmp - test -f tmp - test ! -f tmp.lz4 # must fail (--rm) - test ! -f tmp.lz4.lz4 # must fail (unlz4) - $(PRGDIR)/lz4cat tmp # pass-through mode - test -f tmp - test ! -f tmp.lz4 # must fail (lz4cat) - $(LZ4) tmp tmp.lz4 # creates tmp.lz4 - $(PRGDIR)/lz4cat < tmp.lz4 > tmp3 # checks lz4cat works with stdin (#285) - $(DIFF) -q tmp tmp3 - $(PRGDIR)/lz4cat < tmp > tmp2 # checks lz4cat works with stdin (#285) - $(DIFF) -q tmp tmp2 - cp tmp ./-d - $(LZ4) --rm -- -d -d.lz4 # compresses ./d into ./-d.lz4 + ./datagen -g0 | $(LZ4) -v | $(LZ4) -t + ./datagen -g16KB | $(LZ4) -9 | $(LZ4) -t + ./datagen -g20KB > tmp-tlb-dg20k + $(LZ4) < tmp-tlb-dg20k | $(LZ4) -d > tmp-tlb-dec + $(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec + $(LZ4) --no-frame-crc < tmp-tlb-dg20k | $(LZ4) -d > tmp-tlb-dec + $(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec + ./datagen | $(LZ4) | $(LZ4) -t + ./datagen -g6M -P99 | $(LZ4) -9BD | $(LZ4) -t + ./datagen -g17M | $(LZ4) -9v | $(LZ4) -qt + ./datagen -g33M | $(LZ4) --no-frame-crc | $(LZ4) -t + ./datagen -g256MB | $(LZ4) -vqB4D | $(LZ4) -t + @echo "hello world" > tmp-tlb-hw + $(LZ4) --rm -f tmp-tlb-hw tmp-tlb-hw.lz4 + test ! -f tmp-tlb-hw # must fail (--rm) + test -f tmp-tlb-hw.lz4 + $(PRGDIR)/lz4cat tmp-tlb-hw.lz4 # must display hello world + test -f tmp-tlb-hw.lz4 + $(PRGDIR)/unlz4 --rm tmp-tlb-hw.lz4 tmp-tlb-hw + test -f tmp-tlb-hw + test ! -f tmp-tlb-hw.lz4 # must fail (--rm) + test ! -f tmp-tlb-hw.lz4.lz4 # must fail (unlz4) + $(PRGDIR)/lz4cat tmp-tlb-hw # pass-through mode + test -f tmp-tlb-hw + test ! -f tmp-tlb-hw.lz4 # must fail (lz4cat) + $(LZ4) tmp-tlb-hw tmp-tlb-hw.lz4 # creates tmp-tlb-hw.lz4 + $(PRGDIR)/lz4cat < tmp-tlb-hw.lz4 > tmp-tlb3 # checks lz4cat works with stdin (#285) + $(DIFF) -q tmp-tlb-hw tmp-tlb3 + $(PRGDIR)/lz4cat < tmp-tlb-hw > tmp-tlb2 # checks lz4cat works in pass-through mode + $(DIFF) -q tmp-tlb-hw tmp-tlb2 + cp tmp-tlb-hw ./-d + $(LZ4) --rm -- -d -d.lz4 # compresses ./d into ./-d.lz4 test -f ./-d.lz4 test ! -f ./-d mv ./-d.lz4 ./-z - $(LZ4) -d --rm -- -z tmp4 # uncompresses ./-z into tmp4 + $(LZ4) -d --rm -- -z tmp-tlb4 # uncompresses ./-z into tmp-tlb4 test ! -f ./-z - $(DIFF) -q tmp tmp4 - $(LZ4) -f tmp - cat tmp >> tmp.lz4 - $(LZ4) -f tmp.lz4 # uncompress valid frame followed by invalid data - $(LZ4) -BX tmp -c -q | $(LZ4) -tv # test block checksum - @$(RM) tmp* + $(DIFF) -q tmp-tlb-hw tmp-tlb4 + $(LZ4) -f tmp-tlb-hw + cat tmp-tlb-hw >> tmp-tlb-hw.lz4 + $(LZ4) -f tmp-tlb-hw.lz4 # uncompress valid frame followed by invalid data + $(LZ4) -BX tmp-tlb-hw -c -q | $(LZ4) -tv # test block checksum + @$(RM) tmp-tlb* + +test-lz4-dict: lz4 datagen + @echo "\n ---- test lz4 compression/decompression with dictionary ----" + ./datagen -g16KB > tmp-dict + ./datagen -g32KB > tmp-dict-sample-32k + < tmp-dict-sample-32k $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-32k + ./datagen -g128MB > tmp-dict-sample-128m + < tmp-dict-sample-128m $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-128m + touch tmp-dict-sample-0 + < tmp-dict-sample-0 $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-0 + + < tmp-dict-sample-32k $(LZ4) -D tmp-dict-sample-0 | $(LZ4) -dD tmp-dict-sample-0 | diff - tmp-dict-sample-32k + < tmp-dict-sample-0 $(LZ4) -D tmp-dict-sample-0 | $(LZ4) -dD tmp-dict-sample-0 | diff - tmp-dict-sample-0 + + @echo "\n ---- test lz4 dictionary loading ----" + ./datagen -g128KB > tmp-dict-data-128KB + set -e; \ + for l in 0 1 4 128 32767 32768 32769 65535 65536 65537 98303 98304 98305 131071 131072 131073; do \ + ./datagen -g$$l > tmp-dict-$$l; \ + $(DD) if=tmp-dict-$$l of=tmp-dict-$$l-tail bs=1 count=65536 skip=$$((l > 65536 ? l - 65536 : 0)); \ + < tmp-dict-$$l $(LZ4) -D stdin tmp-dict-data-128KB | $(LZ4) -dD tmp-dict-$$l-tail | $(DIFF) - tmp-dict-data-128KB; \ + < tmp-dict-$$l-tail $(LZ4) -D stdin tmp-dict-data-128KB | $(LZ4) -dD tmp-dict-$$l | $(DIFF) - tmp-dict-data-128KB; \ + done + + @$(RM) tmp-dict* test-lz4-hugefile: lz4 datagen @echo "\n ---- test huge files compression/decompression ----" ./datagen -g6GB | $(LZ4) -vB5D | $(LZ4) -qt ./datagen -g6GB | $(LZ4) -v5BD | $(LZ4) -qt - @$(RM) tmp* test-lz4-testmode: lz4 datagen @echo "\n ---- bench mode ----" - $(LZ4) -bi1 + $(LZ4) -bi1 @echo "\n ---- test mode ----" ! ./datagen | $(LZ4) -t ! ./datagen | $(LZ4) -tf @echo "\n ---- pass-through mode ----" ! ./datagen | $(LZ4) -d > $(VOID) - ./datagen | $(LZ4) -df > $(VOID) - @echo "Hello World !" > tmp1 - $(LZ4) -dcf tmp1 - @echo "from underground..." > tmp2 - $(LZ4) -dcfm tmp1 tmp2 - @echo "\n ---- test cli ----" + ./datagen | $(LZ4) -df > $(VOID) + @echo "Hello World !" > tmp-tlt1 + $(LZ4) -dcf tmp-tlt1 + @echo "from underground..." > tmp-tlt2 + $(LZ4) -dcfm tmp-tlt1 tmp-tlt2 + @echo "\n ---- non-existing source ----" ! $(LZ4) file-does-not-exist ! $(LZ4) -f file-does-not-exist ! $(LZ4) -fm file1-dne file2-dne - ! $(LZ4) -fm file1-dne file2-dne + @$(RM) tmp-tlt test-lz4-opt-parser: lz4 datagen @echo "\n ---- test opt-parser ----" @@ -304,12 +323,13 @@ test-lz4-opt-parser: lz4 datagen ./datagen -g16M -P90 | $(LZ4) -11B5 | $(LZ4) -t ./datagen -g32M -P10 | $(LZ4) -11B5D | $(LZ4) -t -test-lz4: lz4 datagen test-lz4-opt-parser test-lz4-basic test-lz4-multiple test-lz4-sparse \ - test-lz4-frame-concatenation test-lz4-testmode test-lz4-contentSize \ - test-lz4-hugefile +test-lz4: lz4 datagen test-lz4-basic test-lz4-opt-parser test-lz4-multiple \ + test-lz4-sparse test-lz4-frame-concatenation test-lz4-testmode \ + test-lz4-contentSize test-lz4-hugefile test-lz4-dict + @$(RM) tmp* test-lz4c: lz4c datagen - @echo "\n ---- test lz4c version ----" + @echo "\n ---- test lz4c variant ----" ./datagen -g256MB | $(LZ4)c -l -v | $(LZ4)c -t test-lz4c32: CFLAGS+=-m32 @@ -359,25 +379,22 @@ test-frametest: frametest test-frametest32: CFLAGS += -m32 test-frametest32: test-frametest -test-fasttest: fasttest - ./fasttest - test-mem: lz4 datagen fuzzer frametest fullbench @echo "\n ---- valgrind tests : memory analyzer ----" valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID) - ./datagen -g16KB > tmp - valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -BD -f tmp $(VOID) - ./datagen -g16KB -s2 > tmp2 - ./datagen -g16KB -s3 > tmp3 - valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --force --multiple tmp tmp2 tmp3 - ./datagen -g16MB > tmp - valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f tmp tmp2 - valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -t tmp2 - valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 tmp - valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 tmp tmp2 - ./datagen -g256MB > tmp - valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq tmp $(VOID) - $(RM) tmp* + ./datagen -g16KB > ftmdg16K + valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -BD -f ftmdg16K $(VOID) + ./datagen -g16KB -s2 > ftmdg16K2 + ./datagen -g16KB -s3 > ftmdg16K3 + valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --force --multiple ftmdg16K ftmdg16K2 ftmdg16K3 + ./datagen -g16MB > ftmdg16M + valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f ftmdg16M ftmdg16K2 + valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -t ftmdg16K2 + valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 ftmdg16M + valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 ftmdg16M ftmdg16K2 + ./datagen -g256MB > ftmdg256M + valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq ftmdg256M $(VOID) + $(RM) ftm* valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i64 -t1 valgrind --leak-check=yes --error-exitcode=1 ./frametest -i256 diff --git a/tests/fasttest.c b/tests/fasttest.c deleted file mode 100644 index a165293..0000000 --- a/tests/fasttest.c +++ /dev/null @@ -1,131 +0,0 @@ -/************************************** - * Compiler Options - **************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS // for MSVC -# define snprintf sprintf_s -#endif -#ifdef __GNUC__ -# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ -#endif - - -/************************************** - * Includes - **************************************/ -#include <stdio.h> -#include <stdint.h> -#include <stdlib.h> -#include <string.h> -#include "lz4.h" - - -/* Returns non-zero on failure. */ -int test_compress(const char *input, int inSize, char *output, int outSize) -{ - LZ4_stream_t lz4Stream_body = { 0 }; - LZ4_stream_t* lz4Stream = &lz4Stream_body; - - int inOffset = 0; - int outOffset = 0; - - if (inSize & 3) return -1; - - while (inOffset < inSize) { - const int length = inSize >> 2; - if (inSize > 1024) return -2; - if (outSize - (outOffset + 8) < LZ4_compressBound(length)) return -3; - { - const int outBytes = LZ4_compress_fast_continue( - lz4Stream, input + inOffset, output + outOffset + 8, length, outSize-outOffset, 1); - if(outBytes <= 0) return -4; - memcpy(output + outOffset, &length, 4); /* input length */ - memcpy(output + outOffset + 4, &outBytes, 4); /* output length */ - inOffset += length; - outOffset += outBytes + 8; - } - } - if (outOffset + 8 > outSize) return -5; - memset(output + outOffset, 0, 4); - memset(output + outOffset + 4, 0, 4); - return 0; -} - -/* Returns non-zero on failure. Not a safe function. */ -int test_decompress(const char *uncompressed, const char *compressed) -{ - char outBufferA[1024]; - char spacing; /* So prefixEnd != dest */ - char outBufferB[1024]; - char *output = outBufferA; - char *lastOutput = outBufferB; - LZ4_streamDecode_t lz4StreamDecode_body = { 0 }; - LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body; - int offset = 0; - int unOffset = 0; - int lastBytes = 0; - - (void)spacing; - - for(;;) { - int32_t bytes; - int32_t unBytes; - /* Read uncompressed size and compressed size */ - memcpy(&unBytes, compressed + offset, 4); - memcpy(&bytes, compressed + offset + 4, 4); - offset += 8; - /* Check if we reached end of stream or error */ - if(bytes == 0 && unBytes == 0) return 0; - if(bytes <= 0 || unBytes <= 0 || unBytes > 1024) return 1; - - /* Put the last output in the dictionary */ - LZ4_setStreamDecode(lz4StreamDecode, lastOutput, lastBytes); - /* Decompress */ - bytes = LZ4_decompress_fast_continue( - lz4StreamDecode, compressed + offset, output, unBytes); - if(bytes <= 0) return 2; - /* Check result */ - { int const r = memcmp(uncompressed + unOffset, output, unBytes); - if (r) return 3; - } - { char* const tmp = output; output = lastOutput; lastOutput = tmp; } - offset += bytes; - unOffset += unBytes; - lastBytes = unBytes; - } -} - - -int main(int argc, char **argv) -{ - char input[] = - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello!" - "Hello Hello Hello Hello Hello Hello Hello Hello"; - char output[LZ4_COMPRESSBOUND(4096)]; - int r; - - (void)argc; - (void)argv; - - if ((r = test_compress(input, sizeof(input), output, sizeof(output)))) { - return r; - } - if ((r = test_decompress(input, output))) { - return r; - } - return 0; -} diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 2e22912..c134fe3 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -345,10 +345,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); } DISPLAYLEVEL(5, " OK \n"); - } - else + } else { DISPLAYLEVEL(5, " \n"); - } + } } /* Test compression HC destSize */ FUZ_DISPLAYTEST; @@ -359,11 +358,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ctx==NULL, "LZ4_createHC() allocation failed"); compressedBuffer[targetSize] = endCheck; ret = LZ4_compress_HC_destSize(ctx, block, compressedBuffer, &srcSize, targetSize, compressionLevel); + DISPLAYLEVEL(5, "LZ4_compress_HC_destSize(%i): destSize : %7i/%7i; content%7i/%7i ", + compressionLevel, ret, targetSize, srcSize, blockSize); LZ4_freeHC(ctx); FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_HC_destSize() result larger than dst buffer !"); FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_HC_destSize() overwrite dst buffer !"); FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_HC_destSize() fed more than src buffer !"); - DISPLAYLEVEL(5, "destSize : %7i/%7i; content%7i/%7i ", ret, targetSize, srcSize, blockSize); + DISPLAYLEVEL(5, "LZ4_compress_HC_destSize(%i): destSize : %7i/%7i; content%7i/%7i ", + compressionLevel, ret, targetSize, srcSize, blockSize); if (targetSize>0) { /* check correctness */ U32 const crcBase = XXH32(block, srcSize, 0); @@ -376,14 +378,13 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compressHC_destSize"); FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data"); FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe() overwrite dst buffer !"); - { U32 const crcDec = XXH32(decodedBuffer, srcSize, 0); - FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); } - + { U32 const crcDec = XXH32(decodedBuffer, srcSize, 0); + FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); + } DISPLAYLEVEL(5, " OK \n"); - } - else + } else { DISPLAYLEVEL(5, " \n"); - } + } } /* Test compression HC */ FUZ_DISPLAYTEST; @@ -460,8 +461,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c // Test decoding with output size being 10 bytes too short => must fail FUZ_DISPLAYTEST; - if (blockSize>10) - { + if (blockSize>10) { decodedBuffer[blockSize-10] = 0; ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-10); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being 10 bytes too short"); @@ -632,6 +632,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; LZ4_resetStreamHC (&LZ4dictHC, compressionLevel); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); + LZ4_setCompressionLevel(&LZ4dictHC, compressionLevel-1); blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue failed"); @@ -657,7 +658,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); /* Compress HC continue destSize */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST; { int const availableSpace = (FUZ_rand(&randState) % blockSize) + 5; int consumedSize = blockSize; FUZ_DISPLAYTEST; @@ -898,7 +899,6 @@ static void FUZ_unitTests(int compressionLevel) } dict = dst; - //dict = testInput + segStart; dictSize = segSize; dst += segSize + 1; @@ -972,28 +972,27 @@ static void FUZ_unitTests(int compressionLevel) #define BSIZE2 16435 /* first block */ - - messageSize = BSIZE1; - XXH64_update(&xxhOrig, testInput + iNext, messageSize); - crcOrig = XXH64_digest(&xxhOrig); - - result = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); - - result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize); - FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed"); - - XXH64_update(&xxhNew, testVerify + dNext, messageSize); - { U64 const crcNew = XXH64_digest(&xxhNew); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); } - - /* prepare next message */ - dNext += messageSize; - totalMessageSize += messageSize; - messageSize = BSIZE2; - iNext = 132000; - memcpy(testInput + iNext, testInput + 8, messageSize); - if (dNext > dBufferSize) dNext = 0; + messageSize = BSIZE1; + XXH64_update(&xxhOrig, testInput + iNext, messageSize); + crcOrig = XXH64_digest(&xxhOrig); + + result = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); + FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); + + result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize); + FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed"); + + XXH64_update(&xxhNew, testVerify + dNext, messageSize); + { U64 const crcNew = XXH64_digest(&xxhNew); + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); } + + /* prepare next message */ + dNext += messageSize; + totalMessageSize += messageSize; + messageSize = BSIZE2; + iNext = 132000; + memcpy(testInput + iNext, testInput + 8, messageSize); + if (dNext > dBufferSize) dNext = 0; while (totalMessageSize < 9 MB) { XXH64_update(&xxhOrig, testInput + iNext, messageSize); diff --git a/visual/VS2010/datagen/datagen.vcxproj b/visual/VS2010/datagen/datagen.vcxproj index aaf81ad..e24f961 100644 --- a/visual/VS2010/datagen/datagen.vcxproj +++ b/visual/VS2010/datagen/datagen.vcxproj @@ -39,14 +39,14 @@ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <ImportGroup Label="ExtensionSettings"> @@ -91,6 +91,7 @@ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -107,6 +108,7 @@ <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -124,6 +126,7 @@ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -144,6 +147,7 @@ <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> diff --git a/visual/VS2010/frametest/frametest.vcxproj b/visual/VS2010/frametest/frametest.vcxproj index 76d12c9..3196768 100644 --- a/visual/VS2010/frametest/frametest.vcxproj +++ b/visual/VS2010/frametest/frametest.vcxproj @@ -39,14 +39,14 @@ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <ImportGroup Label="ExtensionSettings"> @@ -91,6 +91,7 @@ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -107,6 +108,7 @@ <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -124,6 +126,7 @@ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -144,6 +147,7 @@ <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> diff --git a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj index c10552a..8f503f5 100644 --- a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj +++ b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj @@ -39,14 +39,14 @@ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <ImportGroup Label="ExtensionSettings"> @@ -91,6 +91,7 @@ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -109,6 +110,7 @@ <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -128,6 +130,7 @@ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -150,6 +153,7 @@ <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> diff --git a/visual/VS2010/fullbench/fullbench.vcxproj b/visual/VS2010/fullbench/fullbench.vcxproj index e2d95c9..aa67431 100644 --- a/visual/VS2010/fullbench/fullbench.vcxproj +++ b/visual/VS2010/fullbench/fullbench.vcxproj @@ -39,14 +39,14 @@ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <ImportGroup Label="ExtensionSettings"> @@ -91,6 +91,7 @@ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -107,6 +108,7 @@ <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -124,6 +126,7 @@ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -144,6 +147,7 @@ <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> diff --git a/visual/VS2010/fuzzer/fuzzer.vcxproj b/visual/VS2010/fuzzer/fuzzer.vcxproj index 85d6c9b..21cbf56 100644 --- a/visual/VS2010/fuzzer/fuzzer.vcxproj +++ b/visual/VS2010/fuzzer/fuzzer.vcxproj @@ -39,14 +39,14 @@ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <ImportGroup Label="ExtensionSettings"> @@ -91,6 +91,7 @@ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -107,6 +108,7 @@ <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -124,6 +126,7 @@ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -144,6 +147,7 @@ <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> diff --git a/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj b/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj index 389f13c..56ec3b9 100644 --- a/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj +++ b/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj @@ -40,14 +40,14 @@ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration"> <ConfigurationType>DynamicLibrary</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <ConfigurationType>DynamicLibrary</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <ImportGroup Label="ExtensionSettings"> @@ -96,6 +96,7 @@ <PreprocessorDefinitions>WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <GenerateDebugInformation>true</GenerateDebugInformation> @@ -111,6 +112,7 @@ <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <GenerateDebugInformation>true</GenerateDebugInformation> @@ -127,6 +129,7 @@ <PreprocessorDefinitions>WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <GenerateDebugInformation>true</GenerateDebugInformation> @@ -146,6 +149,7 @@ <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <GenerateDebugInformation>true</GenerateDebugInformation> diff --git a/visual/VS2010/liblz4/liblz4.vcxproj b/visual/VS2010/liblz4/liblz4.vcxproj index a0b8000..61ea159 100644 --- a/visual/VS2010/liblz4/liblz4.vcxproj +++ b/visual/VS2010/liblz4/liblz4.vcxproj @@ -39,14 +39,14 @@ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration"> <ConfigurationType>StaticLibrary</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <ConfigurationType>StaticLibrary</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <ImportGroup Label="ExtensionSettings"> @@ -95,6 +95,7 @@ <PreprocessorDefinitions>WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <GenerateDebugInformation>true</GenerateDebugInformation> @@ -110,6 +111,7 @@ <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <GenerateDebugInformation>true</GenerateDebugInformation> @@ -126,6 +128,7 @@ <PreprocessorDefinitions>WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <GenerateDebugInformation>true</GenerateDebugInformation> @@ -145,6 +148,7 @@ <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <GenerateDebugInformation>true</GenerateDebugInformation> diff --git a/visual/VS2010/lz4/lz4.vcxproj b/visual/VS2010/lz4/lz4.vcxproj index 693e121..de7a714 100644 --- a/visual/VS2010/lz4/lz4.vcxproj +++ b/visual/VS2010/lz4/lz4.vcxproj @@ -39,14 +39,14 @@ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> <CharacterSet>Unicode</CharacterSet> + <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization> </PropertyGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <ImportGroup Label="ExtensionSettings"> @@ -91,6 +91,7 @@ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -108,6 +109,7 @@ <TreatWarningAsError>true</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -126,6 +128,7 @@ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>false</EnablePREfast> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -147,6 +150,7 @@ <TreatWarningAsError>false</TreatWarningAsError> <EnablePREfast>true</EnablePREfast> <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions> + <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary> </ClCompile> <Link> <SubSystem>Console</SubSystem> |