diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | Makefile | 10 | ||||
| -rw-r--r-- | NEWS | 4 | ||||
| -rw-r--r-- | contrib/cmake_unofficial/CMakeLists.txt | 24 | ||||
| -rw-r--r-- | doc/lz4_manual.html | 68 | ||||
| -rw-r--r-- | examples/frameCompress.c | 249 | ||||
| -rw-r--r-- | lib/lz4.c | 132 | ||||
| -rw-r--r-- | lib/lz4.h | 195 | ||||
| -rw-r--r-- | lib/lz4frame.c | 108 | ||||
| -rw-r--r-- | lib/lz4frame.h | 5 | ||||
| -rw-r--r-- | lib/lz4frame_static.h | 2 | ||||
| -rw-r--r-- | lib/lz4hc.c | 124 | ||||
| -rw-r--r-- | lib/lz4hc.h | 234 | ||||
| -rw-r--r-- | programs/lz4.1 | 2 | ||||
| -rw-r--r-- | programs/lz4cli.c | 1 | ||||
| -rw-r--r-- | tests/Makefile | 2 | ||||
| -rw-r--r-- | tests/fasttest.c | 4 | ||||
| -rw-r--r-- | tests/frametest.c | 297 | ||||
| -rw-r--r-- | tests/fullbench.c | 36 | ||||
| -rw-r--r-- | tests/fuzzer.c | 323 |
20 files changed, 1050 insertions, 771 deletions
@@ -27,3 +27,4 @@ _codelite_lz4/ # Mac .DS_Store +*.dSYM @@ -135,16 +135,16 @@ cmake: @cd contrib/cmake_unofficial; cmake $(CMAKE_PARAMS) CMakeLists.txt; $(MAKE) gpptest: clean - $(MAKE) all CC=g++ CFLAGS="-O3 -I../lib -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" + CC=g++ $(MAKE) all CFLAGS="-O3 -I../lib -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" c_standards: clean - $(MAKE) all MOREFLAGS="-std=gnu90 -Werror" + $(MAKE) all MOREFLAGS="-std=gnu90 -Werror" $(MAKE) clean - $(MAKE) all MOREFLAGS="-std=c99 -Werror" + $(MAKE) all MOREFLAGS="-std=c99 -Werror" $(MAKE) clean - $(MAKE) all MOREFLAGS="-std=gnu99 -Werror" + $(MAKE) all MOREFLAGS="-std=gnu99 -Werror" $(MAKE) clean - $(MAKE) all MOREFLAGS="-std=c11 -Werror" + $(MAKE) all MOREFLAGS="-std=c11 -Werror" $(MAKE) clean endif @@ -8,7 +8,7 @@ Fix : better ratio on 64-bits big-endian targets Improved cmake build script, by Evan Nemerson New liblz4-dll project, by Przemyslaw Skibinki Makefile: Generates object files (*.o) for faster (re)compilation on low power systems -cli : new : --rm command +cli : new : --rm and --help commands cli : new : preserved file attributes, by Przemyslaw Skibinki cli : fix : crash on some invalid inputs cli : fix : -t correctly validates lz4-compressed files, by Nick Terrell @@ -16,7 +16,7 @@ cli : fix : detects and reports fread() errors, thanks to Hiroshi Fujishima repo cli : bench : new : -r recursive mode lz4cat : can cat multiple files in a single command line (#184) Added : doc/lz4_manual.html, by Przemyslaw Skibinski -Added : dictionary compression example, by Nick Terrell +Added : dictionary compression and frame decompression examples, by Nick Terrell Added : Debianization, by Evgeniy Polyakov r131 diff --git a/contrib/cmake_unofficial/CMakeLists.txt b/contrib/cmake_unofficial/CMakeLists.txt index 9324338..de070d6 100644 --- a/contrib/cmake_unofficial/CMakeLists.txt +++ b/contrib/cmake_unofficial/CMakeLists.txt @@ -1,3 +1,15 @@ +# CMake support for LZ4 +# +# To the extent possible under law, the author(s) have dedicated all +# copyright and related and neighboring rights to this software to +# the public domain worldwide. This software is distributed without +# any warranty. +# +# For details, see <http://creativecommons.org/publicdomain/zero/1.0/>. +# +# LZ4's CMake support is maintained by Evan Nemerson; when filing +# bugs please mention @nemequ to make sure I see it. + set(LZ4_TOP_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..") # Parse version information @@ -74,12 +86,22 @@ set(LZ4_CLI_SOURCES "${LZ4_PROG_SOURCE_DIR}/lz4io.c" "${LZ4_PROG_SOURCE_DIR}/datagen.c") +# Whether to use position independent code for the static library. If +# we're building a shared library this is ignored and PIC is always +# used. +option(LZ4_POSITION_INDEPENDENT_LIB "Use position independent code for static library (if applicable)" ON) +if(LZ4_POSITION_INDEPENDENT_LIB OR BUILD_SHARED_LIBS) + set(LZ4_POSITION_INDEPENDENT_CODE TRUE) +else() + set(LZ4_POSITION_INDEPENDENT_CODE FALSE) +endif() + # liblz4 add_library(lz4 ${LZ4_SOURCES}) set_target_properties(lz4 PROPERTIES SOVERSION "${LZ4_VERSION_STRING}" VERSION "${LZ4_VERSION_STRING}" - POSITION_INDEPENDENT_CODE TRUE) + POSITION_INDEPENDENT_CODE ${LZ4_POSITION_INDEPENDENT_CODE}) # lz4 add_executable(lz4cli ${LZ4_CLI_SOURCES}) diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html index 838dbf4..bc46645 100644 --- a/doc/lz4_manual.html +++ b/doc/lz4_manual.html @@ -10,10 +10,11 @@ <ol> <li><a href="#Chapter1">Introduction</a></li> <li><a href="#Chapter2">Tuning parameter</a></li> -<li><a href="#Chapter3">Simple Functions</a></li> -<li><a href="#Chapter4">Advanced Functions</a></li> -<li><a href="#Chapter5">Streaming Compression Functions</a></li> -<li><a href="#Chapter6">Streaming Decompression Functions</a></li> +<li><a href="#Chapter3">Private definitions</a></li> +<li><a href="#Chapter4">Simple Functions</a></li> +<li><a href="#Chapter5">Advanced Functions</a></li> +<li><a href="#Chapter6">Streaming Compression Functions</a></li> +<li><a href="#Chapter7">Streaming Decompression Functions</a></li> </ol> <hr> <a name="Chapter1"></a><h2>Introduction</h2><pre> @@ -48,7 +49,45 @@ const char* LZ4_versionString (void); </p></pre><BR> -<a name="Chapter3"></a><h2>Simple Functions</h2><pre></pre> +<a name="Chapter3"></a><h2>Private definitions</h2><pre> + Do not use these definitions. + They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. + If you use these definitions in your code, it will break when you upgrade LZ4 to a new version. +<BR></pre> + +<pre><b>typedef struct { + uint32_t hashTable[LZ4_HASH_SIZE_U32]; + uint32_t currentOffset; + uint32_t initCheck; + const uint8_t* dictionary; + uint8_t* bufferStart; </b>/* obsolete, used for slideInputBuffer */<b> + uint32_t dictSize; +} LZ4_stream_t_internal; +</b></pre><BR> +<pre><b>typedef struct { + const uint8_t* externalDict; + size_t extDictSize; + const uint8_t* prefixEnd; + size_t prefixSize; +} LZ4_streamDecode_t_internal; +</b></pre><BR> +<pre><b>typedef struct { + unsigned int hashTable[LZ4_HASH_SIZE_U32]; + unsigned int currentOffset; + unsigned int initCheck; + const unsigned char* dictionary; + unsigned char* bufferStart; </b>/* obsolete, used for slideInputBuffer */<b> + unsigned int dictSize; +} LZ4_stream_t_internal; +</b></pre><BR> +<pre><b>typedef struct { + const unsigned char* externalDict; + size_t extDictSize; + const unsigned char* prefixEnd; + size_t prefixSize; +} LZ4_streamDecode_t_internal; +</b></pre><BR> +<a name="Chapter4"></a><h2>Simple Functions</h2><pre></pre> <pre><b>int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize); </b><p> Compresses 'sourceSize' bytes from buffer 'source' @@ -75,7 +114,7 @@ const char* LZ4_versionString (void); It never writes outside output buffer, nor reads outside input buffer. </p></pre><BR> -<a name="Chapter4"></a><h2>Advanced Functions</h2><pre></pre> +<a name="Chapter5"></a><h2>Advanced Functions</h2><pre></pre> <pre><b>int LZ4_compressBound(int inputSize); </b><p> Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) @@ -137,9 +176,14 @@ int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets </p></pre><BR> -<a name="Chapter5"></a><h2>Streaming Compression Functions</h2><pre></pre> +<a name="Chapter6"></a><h2>Streaming Compression Functions</h2><pre></pre> -<pre><b>typedef struct { long long table[LZ4_STREAMSIZE_U64]; } LZ4_stream_t; +<pre><b>typedef struct { + union { + long long table[LZ4_STREAMSIZE_U64]; + LZ4_stream_t_internal internal_donotuse; + }; +} LZ4_stream_t; </b><p> information structure to track an LZ4 stream. important : init this structure content before first use ! note : only allocated directly the structure if you are statically linking LZ4 @@ -187,9 +231,13 @@ int LZ4_freeStream (LZ4_stream_t* streamPtr); </p></pre><BR> -<a name="Chapter6"></a><h2>Streaming Decompression Functions</h2><pre></pre> +<a name="Chapter7"></a><h2>Streaming Decompression Functions</h2><pre></pre> -<pre><b>typedef struct { unsigned long long table[LZ4_STREAMDECODESIZE_U64]; } LZ4_streamDecode_t; +<pre><b>typedef struct { + union { + unsigned long long table[LZ4_STREAMDECODESIZE_U64]; + LZ4_streamDecode_t_internal internal_donotuse; + }; </b></pre><BR> <pre><b>LZ4_streamDecode_t* LZ4_createStreamDecode(void); int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); diff --git a/examples/frameCompress.c b/examples/frameCompress.c index b52b042..b7a3576 100644 --- a/examples/frameCompress.c +++ b/examples/frameCompress.c @@ -8,7 +8,7 @@ #include <lz4frame.h> -#define BUF_SIZE (16*1024) +#define BUF_SIZE 16*1024 #define LZ4_HEADER_SIZE 19 #define LZ4_FOOTER_SIZE 4 @@ -19,7 +19,7 @@ static const LZ4F_preferences_t lz4_preferences = { { 0, 0, 0, 0 }, /* reserved, must be set to 0 */ }; -static int compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) { +static size_t compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) { LZ4F_errorCode_t r; LZ4F_compressionContext_t ctx; char *src, *buf = NULL; @@ -27,14 +27,14 @@ static int compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) r = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); if (LZ4F_isError(r)) { - printf("Failed to create context: error %zu", r); + printf("Failed to create context: error %zu\n", r); return 1; } r = 1; src = malloc(BUF_SIZE); if (!src) { - printf("Not enough memory"); + printf("Not enough memory\n"); goto cleanup; } @@ -42,13 +42,13 @@ static int compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) size = frame_size + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE; buf = malloc(size); if (!buf) { - printf("Not enough memory"); + printf("Not enough memory\n"); goto cleanup; } n = offset = count_out = LZ4F_compressBegin(ctx, buf, size, &lz4_preferences); if (LZ4F_isError(n)) { - printf("Failed to start compression: error %zu", n); + printf("Failed to start compression: error %zu\n", n); goto cleanup; } @@ -62,7 +62,7 @@ static int compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) n = LZ4F_compressUpdate(ctx, buf + offset, size - offset, src, k, NULL); if (LZ4F_isError(n)) { - printf("Compression failed: error %zu", n); + printf("Compression failed: error %zu\n", n); goto cleanup; } @@ -74,9 +74,9 @@ static int compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) k = fwrite(buf, 1, offset, out); if (k < offset) { if (ferror(out)) - printf("Write failed"); + printf("Write failed\n"); else - printf("Short write"); + printf("Short write\n"); goto cleanup; } @@ -86,7 +86,7 @@ static int compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) n = LZ4F_compressEnd(ctx, buf + offset, size - offset, NULL); if (LZ4F_isError(n)) { - printf("Failed to end compression: error %zu", n); + printf("Failed to end compression: error %zu\n", n); goto cleanup; } @@ -97,9 +97,9 @@ static int compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) k = fwrite(buf, 1, offset, out); if (k < offset) { if (ferror(out)) - printf("Write failed"); + printf("Write failed\n"); else - printf("Short write"); + printf("Short write\n"); goto cleanup; } @@ -114,56 +114,205 @@ static int compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) return r; } -static int compress(const char *input, const char *output) { - char *tmp = NULL; - FILE *in = NULL, *out = NULL; - size_t size_in = 0, size_out = 0; - int r = 1; - - if (!output) { - size_t len = strlen(input); - - output = tmp = malloc(len + 5); - if (!tmp) { - printf("Not enough memory"); - return 1; - } - strcpy(tmp, input); - strcpy(tmp + len, ".lz4"); +static size_t get_block_size(const LZ4F_frameInfo_t* info) { + switch (info->blockSizeID) { + case LZ4F_max64KB: return 1 << 16; + case LZ4F_max256KB: return 1 << 18; + case LZ4F_max1MB: return 1 << 20; + case LZ4F_max4MB: return 1 << 22; + default: + printf("Impossible unless more block sizes are allowed\n"); + exit(1); } +} + +static size_t decompress_file(FILE *in, FILE *out) { + void * const src = malloc(BUF_SIZE); + void *dst = NULL; + void *srcPtr = src; + void *srcEnd = src; + size_t srcSize = 0; + size_t dstSize = 0; + size_t dstCapacity = 0; + LZ4F_dctx *dctx = NULL; + size_t ret; - in = fopen(input, "rb"); - if (!in) { - fprintf(stderr, "Failed to open input file %s: %s\n", input, strerror(errno)); + /* Initialization */ + ret = LZ4F_createDecompressionContext(&dctx, 100); + if (LZ4F_isError(ret)) { + printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(ret)); goto cleanup; } - out = fopen(output, "wb"); - if (!out) { - fprintf(stderr, "Failed to open output file %s: %s\n", output, strerror(errno)); + /* Decompression */ + ret = 1; + while (ret != 0) { + /* INVARIANT: At this point srcPtr == srcEnd */ + /* Load more input */ + srcSize = fread(src, 1, BUF_SIZE, in); + if (srcSize == 0 || ferror(in)) { + printf("Decompress: not enough input or error reading file\n"); + goto cleanup; + } + srcPtr = src; + srcEnd = srcPtr + srcSize; + /* Allocate destination buffer if it isn't already */ + if (!dst) { + LZ4F_frameInfo_t info; + ret = LZ4F_getFrameInfo(dctx, &info, src, &srcSize); + if (LZ4F_isError(ret)) { + printf("LZ4F_getFrameInfo error: %s\n", LZ4F_getErrorName(ret)); + goto cleanup; + } + /* Allocating enough space for an entire block isn't necessary for + * correctness, but it allows some memcpy's to be elided. + */ + dstCapacity = get_block_size(&info); + dst = malloc(dstCapacity); + srcPtr += srcSize; + srcSize = srcEnd - srcPtr; + } + /* Decompress: + * Continue while there is more input to read and the frame isn't over. + * If srcPtr == srcEnd then we know that there is no more output left in the + * internal buffer left to flush. + */ + while (srcPtr != srcEnd && ret != 0) { + /* INVARIANT: Any data left in dst has already been written */ + dstSize = dstCapacity; + ret = LZ4F_decompress(dctx, dst, &dstSize, srcPtr, &srcSize, /* LZ4F_decompressOptions_t */ NULL); + if (LZ4F_isError(ret)) { + printf("Decompression error: %s\n", LZ4F_getErrorName(ret)); + goto cleanup; + } + /* Flush output */ + if (dstSize != 0){ + size_t written = fwrite(dst, 1, dstSize, out); + printf("Writing %zu bytes\n", dstSize); + if (written != dstSize) { + printf("Decompress: Failed to write to file\n"); + ret = 1; + goto cleanup; + } + } + /* Update input */ + srcPtr += srcSize; + srcSize = srcEnd - srcPtr; + } + } + /* Check that there isn't trailing input data after the frame. + * It is valid to have multiple frames in the same file, but this example + * doesn't support it. + */ + ret = fread(src, 1, 1, in); + if (ret != 0 || !feof(in)) { + printf("Decompress: Trailing data left in file after frame\n"); goto cleanup; } - r = compress_file(in, out, &size_in, &size_out); - if (r == 0) - printf("%s: %zu → %zu bytes, %.1f%%\n", - input, size_in, size_out, - (double)size_out / size_in * 100); - cleanup: - if (in) - fclose(in); - if (out) - fclose(out); - free(tmp); - return r; +cleanup: + free(src); + if (dst) free(dst); + if (dctx) ret = LZ4F_freeDecompressionContext(dctx); + return ret; } +int compare(FILE* fp0, FILE* fp1) +{ + int result = 0; + + while(0 == result) { + char b0[1024]; + char b1[1024]; + const size_t r0 = fread(b0, 1, sizeof(b0), fp0); + const size_t r1 = fread(b1, 1, sizeof(b1), fp1); + + result = (int) r0 - (int) r1; + + if (0 == r0 || 0 == r1) { + break; + } + if (0 == result) { + result = memcmp(b0, b1, r0); + } + } + + return result; +} int main(int argc, char **argv) { - if (argc < 2 || argc > 3) { - fprintf(stderr, "Syntax: %s <input> <output>\n", argv[0]); - return EXIT_FAILURE; + char inpFilename[256] = { 0 }; + char lz4Filename[256] = { 0 }; + char decFilename[256] = { 0 }; + + if(argc < 2) { + printf("Please specify input filename\n"); + return 0; } - return compress(argv[1], argv[2]); + snprintf(inpFilename, 256, "%s", argv[1]); + snprintf(lz4Filename, 256, "%s.lz4", argv[1]); + snprintf(decFilename, 256, "%s.lz4.dec", argv[1]); + + printf("inp = [%s]\n", inpFilename); + printf("lz4 = [%s]\n", lz4Filename); + printf("dec = [%s]\n", decFilename); + + /* compress */ + { + FILE* inpFp = fopen(inpFilename, "rb"); + FILE* outFp = fopen(lz4Filename, "wb"); + size_t sizeIn = 0; + size_t sizeOut = 0; + size_t ret; + + printf("compress : %s -> %s\n", inpFilename, lz4Filename); + ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut); + if (ret) { + printf("compress : failed with code %zu\n", ret); + return ret; + } + printf("%s: %zu → %zu bytes, %.1f%%\n", + inpFilename, sizeIn, sizeOut, + (double)sizeOut / sizeIn * 100); + printf("compress : done\n"); + + fclose(outFp); + fclose(inpFp); + } + + /* decompress */ + { + FILE* inpFp = fopen(lz4Filename, "rb"); + FILE* outFp = fopen(decFilename, "wb"); + size_t ret; + + printf("decompress : %s -> %s\n", lz4Filename, decFilename); + ret = decompress_file(inpFp, outFp); + if (ret) { + printf("decompress : failed with code %zu\n", ret); + return ret; + } + printf("decompress : done\n"); + + fclose(outFp); + fclose(inpFp); + } + + /* verify */ + { + FILE* inpFp = fopen(inpFilename, "rb"); + FILE* decFp = fopen(decFilename, "rb"); + + printf("verify : %s <-> %s\n", inpFilename, decFilename); + const int cmp = compare(inpFp, decFp); + if(0 == cmp) { + printf("verify : OK\n"); + } else { + printf("verify : NG\n"); + } + + fclose(decFp); + fclose(inpFp); + } } @@ -41,7 +41,9 @@ * Select how default compression functions will allocate memory for their hash table, * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). */ -#define HEAPMODE 0 +#ifndef HEAPMODE +# define HEAPMODE 0 +#endif /* * ACCELERATION_DEFAULT : @@ -109,8 +111,7 @@ # endif #endif /* _MSC_VER */ -/* LZ4_GCC_VERSION is defined into lz4.h */ -#if (LZ4_GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__) +#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) # define expect(expr,value) (__builtin_expect ((expr),(value)) ) #else # define expect(expr,value) (expr) @@ -299,7 +300,7 @@ static unsigned LZ4_NbCommonBytes (register size_t val) unsigned long r = 0; _BitScanForward64( &r, (U64)val ); return (int)(r>>3); -# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT) +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) return (__builtin_ctzll((U64)val) >> 3); # else static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; @@ -310,7 +311,7 @@ static unsigned LZ4_NbCommonBytes (register size_t val) unsigned long r; _BitScanForward( &r, (U32)val ); return (int)(r>>3); -# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT) +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) return (__builtin_ctz((U32)val) >> 3); # else static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; @@ -323,7 +324,7 @@ static unsigned LZ4_NbCommonBytes (register size_t val) unsigned long r = 0; _BitScanReverse64( &r, val ); return (unsigned)(r>>3); -# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT) +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) return (__builtin_clzll((U64)val) >> 3); # else unsigned r; @@ -337,7 +338,7 @@ static unsigned LZ4_NbCommonBytes (register size_t val) unsigned long r = 0; _BitScanReverse( &r, (unsigned long)val ); return (unsigned)(r>>3); -# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT) +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) return (__builtin_clz((U32)val) >> 3); # else unsigned r; @@ -371,10 +372,6 @@ static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLi /*-************************************ * Local Constants **************************************/ -#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) -#define HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) -#define HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ - static const int LZ4_64Klimit = ((64 KB) + (MFLIMIT-1)); static const U32 LZ4_skipTrigger = 6; /* Increase this value ==> compression run slower on incompressible data */ @@ -382,15 +379,6 @@ static const U32 LZ4_skipTrigger = 6; /* Increase this value ==> compression ru /*-************************************ * Local Structures and types **************************************/ -typedef struct { - U32 hashTable[HASH_SIZE_U32]; - U32 currentOffset; - U32 initCheck; - const BYTE* dictionary; - BYTE* bufferStart; /* obsolete, used for slideInputBuffer */ - U32 dictSize; -} LZ4_stream_t_internal; - typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive; typedef enum { byPtr, byU32, byU16 } tableType_t; @@ -470,7 +458,7 @@ FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableTy /** LZ4_compress_generic() : inlined, to ensure branches are decided at compilation time */ FORCE_INLINE int LZ4_compress_generic( - void* const ctx, + LZ4_stream_t_internal* const dictPtr, const char* const source, char* const dest, const int inputSize, @@ -481,8 +469,6 @@ FORCE_INLINE int LZ4_compress_generic( const dictIssue_directive dictIssue, const U32 acceleration) { - LZ4_stream_t_internal* const dictPtr = (LZ4_stream_t_internal*)ctx; - const BYTE* ip = (const BYTE*) source; const BYTE* base; const BYTE* lowLimit; @@ -523,7 +509,7 @@ FORCE_INLINE int LZ4_compress_generic( if (inputSize<LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ /* First Byte */ - LZ4_putPosition(ip, ctx, tableType, base); + LZ4_putPosition(ip, dictPtr->hashTable, tableType, base); ip++; forwardH = LZ4_hashPosition(ip, tableType); /* Main Loop */ @@ -543,7 +529,7 @@ FORCE_INLINE int LZ4_compress_generic( if (unlikely(forwardIp > mflimit)) goto _last_literals; - match = LZ4_getPositionOnHash(h, ctx, tableType, base); + match = LZ4_getPositionOnHash(h, dictPtr->hashTable, tableType, base); if (dict==usingExtDict) { if (match < (const BYTE*)source) { refDelta = dictDelta; @@ -553,7 +539,7 @@ FORCE_INLINE int LZ4_compress_generic( lowLimit = (const BYTE*)source; } } forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, ctx, tableType, base); + LZ4_putPositionOnHash(ip, h, dictPtr->hashTable, tableType, base); } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) @@ -626,10 +612,10 @@ _next_match: if (ip > mflimit) break; /* Fill table */ - LZ4_putPosition(ip-2, ctx, tableType, base); + LZ4_putPosition(ip-2, dictPtr->hashTable, tableType, base); /* Test next position */ - match = LZ4_getPosition(ip, ctx, tableType, base); + match = LZ4_getPosition(ip, dictPtr->hashTable, tableType, base); if (dict==usingExtDict) { if (match < (const BYTE*)source) { refDelta = dictDelta; @@ -638,7 +624,7 @@ _next_match: refDelta = 0; lowLimit = (const BYTE*)source; } } - LZ4_putPosition(ip, ctx, tableType, base); + LZ4_putPosition(ip, dictPtr->hashTable, tableType, base); if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) && (match+MAX_DISTANCE>=ip) && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) @@ -673,19 +659,20 @@ _last_literals: int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { + LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse; LZ4_resetStream((LZ4_stream_t*)state); if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; if (maxOutputSize >= LZ4_compressBound(inputSize)) { if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); else - return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); } else { if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); else - return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); } } @@ -722,9 +709,9 @@ int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int m LZ4_resetStream(&ctx); if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(&ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); else - return LZ4_compress_generic(&ctx, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); } @@ -733,7 +720,7 @@ int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int m ********************************/ static int LZ4_compress_destSize_generic( - void* const ctx, + LZ4_stream_t_internal* const ctx, const char* const src, char* const dst, int* const srcSizePtr, @@ -765,7 +752,7 @@ static int LZ4_compress_destSize_generic( /* First Byte */ *srcSizePtr = 0; - LZ4_putPosition(ip, ctx, tableType, base); + LZ4_putPosition(ip, ctx->hashTable, tableType, base); ip++; forwardH = LZ4_hashPosition(ip, tableType); /* Main Loop */ @@ -786,9 +773,9 @@ static int LZ4_compress_destSize_generic( if (unlikely(forwardIp > mflimit)) goto _last_literals; - match = LZ4_getPositionOnHash(h, ctx, tableType, base); + match = LZ4_getPositionOnHash(h, ctx->hashTable, tableType, base); forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, ctx, tableType, base); + LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base); } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) || (LZ4_read32(match) != LZ4_read32(ip)) ); @@ -847,11 +834,11 @@ _next_match: if (op > oMaxSeq) break; /* Fill table */ - LZ4_putPosition(ip-2, ctx, tableType, base); + LZ4_putPosition(ip-2, ctx->hashTable, tableType, base); /* Test next position */ - match = LZ4_getPosition(ip, ctx, tableType, base); - LZ4_putPosition(ip, ctx, tableType, base); + match = LZ4_getPosition(ip, ctx->hashTable, tableType, base); + LZ4_putPosition(ip, ctx->hashTable, tableType, base); if ( (match+MAX_DISTANCE>=ip) && (LZ4_read32(match)==LZ4_read32(ip)) ) { token=op++; *token=0; goto _next_match; } @@ -888,17 +875,17 @@ _last_literals: } -static int LZ4_compress_destSize_extState (void* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize) +static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize) { - LZ4_resetStream((LZ4_stream_t*)state); + LZ4_resetStream(state); if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); } else { if (*srcSizePtr < LZ4_64Klimit) - return LZ4_compress_destSize_generic(state, src, dst, srcSizePtr, targetDstSize, byU16); + return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16); else - return LZ4_compress_destSize_generic(state, src, dst, srcSizePtr, targetDstSize, LZ4_64bits() ? byU32 : byPtr); + return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, LZ4_64bits() ? byU32 : byPtr); } } @@ -906,10 +893,10 @@ static int LZ4_compress_destSize_extState (void* state, const char* src, char* d int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) { #if (HEAPMODE) - void* ctx = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ #else LZ4_stream_t ctxBody; - void* ctx = &ctxBody; + LZ4_stream_t* ctx = &ctxBody; #endif int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); @@ -949,7 +936,7 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_stream) #define HASH_UNIT sizeof(size_t) int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) { - LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict; + LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse; const BYTE* p = (const BYTE*)dictionary; const BYTE* const dictEnd = p + dictSize; const BYTE* base; @@ -987,7 +974,7 @@ static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) U32 const delta = LZ4_dict->currentOffset - 64 KB; const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; int i; - for (i=0; i<HASH_SIZE_U32; i++) { + for (i=0; i<LZ4_HASH_SIZE_U32; i++) { if (LZ4_dict->hashTable[i] < delta) LZ4_dict->hashTable[i]=0; else LZ4_dict->hashTable[i] -= delta; } @@ -1000,7 +987,7 @@ static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { - LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_stream; + LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse; const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; const BYTE* smallest = (const BYTE*) source; @@ -1023,9 +1010,9 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch if (dictEnd == (const BYTE*)source) { int result; if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); else - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); streamPtr->dictSize += (U32)inputSize; streamPtr->currentOffset += (U32)inputSize; return result; @@ -1034,9 +1021,9 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch /* external dictionary mode */ { int result; if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); else - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); streamPtr->dictionary = (const BYTE*)source; streamPtr->dictSize = (U32)inputSize; streamPtr->currentOffset += (U32)inputSize; @@ -1048,15 +1035,15 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch /* Hidden debug function, to force external dictionary mode */ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) { - LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_dict; + LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse; int result; const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; const BYTE* smallest = dictEnd; if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; - LZ4_renormDictT((LZ4_stream_t_internal*)LZ4_dict, smallest); + LZ4_renormDictT(streamPtr, smallest); - result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); streamPtr->dictionary = (const BYTE*)source; streamPtr->dictSize = (U32)inputSize; @@ -1075,7 +1062,7 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* */ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) { - LZ4_stream_t_internal* const dict = (LZ4_stream_t_internal*) LZ4_dict; + LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ @@ -1280,13 +1267,6 @@ int LZ4_decompress_fast(const char* source, char* dest, int originalSize) /*===== streaming decompression functions =====*/ -typedef struct { - const BYTE* externalDict; - size_t extDictSize; - const BYTE* prefixEnd; - size_t prefixSize; -} LZ4_streamDecode_t_internal; - /* * If you prefer dynamic allocation methods, * LZ4_createStreamDecode() @@ -1313,7 +1293,7 @@ int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) */ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) { - LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; lz4sd->prefixSize = (size_t) dictSize; lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; lz4sd->externalDict = NULL; @@ -1330,7 +1310,7 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti */ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) { - LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; int result; if (lz4sd->prefixEnd == (BYTE*)dest) { @@ -1356,7 +1336,7 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) { - LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; int result; if (lz4sd->prefixEnd == (BYTE*)dest) { @@ -1442,29 +1422,29 @@ int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } -static void LZ4_init(LZ4_stream_t_internal* lz4ds, BYTE* base) +static void LZ4_init(LZ4_stream_t* lz4ds, BYTE* base) { - MEM_INIT(lz4ds, 0, LZ4_STREAMSIZE); - lz4ds->bufferStart = base; + MEM_INIT(lz4ds, 0, sizeof(LZ4_stream_t)); + lz4ds->internal_donotuse.bufferStart = base; } int LZ4_resetStreamState(void* state, char* inputBuffer) { if ((((size_t)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ - LZ4_init((LZ4_stream_t_internal*)state, (BYTE*)inputBuffer); + LZ4_init((LZ4_stream_t*)state, (BYTE*)inputBuffer); return 0; } void* LZ4_create (char* inputBuffer) { - void* lz4ds = ALLOCATOR(8, LZ4_STREAMSIZE_U64); - LZ4_init ((LZ4_stream_t_internal*)lz4ds, (BYTE*)inputBuffer); + LZ4_stream_t* lz4ds = (LZ4_stream_t*)ALLOCATOR(8, sizeof(LZ4_stream_t)); + LZ4_init (lz4ds, (BYTE*)inputBuffer); return lz4ds; } char* LZ4_slideInputBuffer (void* LZ4_Data) { - LZ4_stream_t_internal* ctx = (LZ4_stream_t_internal*)LZ4_Data; + LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)LZ4_Data)->internal_donotuse; int dictSize = LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB); return (char*)(ctx->bufferStart + dictSize); } @@ -39,6 +39,10 @@ extern "C" { #endif +/* --- Dependency --- */ +#include <stddef.h> /* size_t */ + + /** Introduction @@ -53,10 +57,12 @@ extern "C" { - unbounded multiple steps (described as Streaming compression) lz4.h provides block compression functions. It gives full buffer control to user. - Block compression functions are not-enough to send information, - since it's still necessary to provide metadata (such as compressed size), - and each application can do it in whichever way it wants. - For interoperability, there is LZ4 frame specification (doc/lz4_Frame_format.md). + Decompressing an lz4-compressed block also requires metadata (such as compressed size). + Each application is free to encode such metadata in whichever way it wants. + + An additional format, called LZ4 frame specification (doc/lz4_Frame_format.md), + take care of encoding standard metadata alongside LZ4-compressed blocks. + If your application requires interoperability, it's recommended to use it. A library is provided to take care of it, see lz4frame.h. */ @@ -77,9 +83,6 @@ extern "C" { /*========== Version =========== */ -LZ4LIB_API int LZ4_versionNumber (void); -LZ4LIB_API const char* LZ4_versionString (void); - #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ #define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */ #define LZ4_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */ @@ -91,6 +94,9 @@ LZ4LIB_API const char* LZ4_versionString (void); #define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) #define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) +LZ4LIB_API int LZ4_versionNumber (void); +LZ4LIB_API const char* LZ4_versionString (void); + /*-************************************ * Tuning parameter @@ -220,31 +226,21 @@ LZ4LIB_API int LZ4_decompress_safe_partial (const char* source, char* dest, int /*-********************************************* * Streaming Compression Functions ***********************************************/ -#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) -#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(long long)) -/*! - * LZ4_stream_t : - * information structure to track an LZ4 stream. - * important : init this structure content before first use ! - * note : only allocated directly the structure if you are statically linking LZ4 - * If you are using liblz4 as a DLL, please use below construction methods instead. - */ -typedef struct { long long table[LZ4_STREAMSIZE_U64]; } LZ4_stream_t; - -/*! LZ4_resetStream() : - * Use this function to init an allocated `LZ4_stream_t` structure - */ -LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); +typedef struct LZ4_stream_s LZ4_stream_t; /* incomplete type (defined later) */ /*! LZ4_createStream() and LZ4_freeStream() : * LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure. * LZ4_freeStream() releases its memory. - * In the context of a DLL (liblz4), please use these methods rather than the static struct. - * They are more future proof, in case of a change of `LZ4_stream_t` size. */ LZ4LIB_API LZ4_stream_t* LZ4_createStream(void); LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); +/*! LZ4_resetStream() : + * An LZ4_stream_t structure can be allocated once and re-used multiple times. + * Use this function to init an allocated `LZ4_stream_t` structure and start a new compression. + */ +LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); + /*! LZ4_loadDict() : * Use this function to load a static dictionary into LZ4_stream. * Any previous data will be forgotten, only 'dictionary' will remain in memory. @@ -274,21 +270,11 @@ LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dict /*-********************************************** * Streaming Decompression Functions +* Bufferless synchronous API ************************************************/ +typedef struct LZ4_streamDecode_s LZ4_streamDecode_t; /* incomplete type (defined later) */ -#define LZ4_STREAMDECODESIZE_U64 4 -#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) -typedef struct { unsigned long long table[LZ4_STREAMDECODESIZE_U64]; } LZ4_streamDecode_t; -/*! - * LZ4_streamDecode_t - * information structure to track an LZ4 stream. - * init this structure content using LZ4_setStreamDecode or memset() before first use ! - * - * In the context of a DLL (liblz4) please prefer usage of construction methods below. - * They are more future proof, in case of a change of LZ4_streamDecode_t size in the future. - * LZ4_createStreamDecode will allocate and initialize an LZ4_streamDecode_t structure - * LZ4_freeStreamDecode releases its memory. - */ +/* creation / destruction of streaming decompression tracking structure */ LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); @@ -300,7 +286,7 @@ LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_str LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); /*! -*_continue() : +LZ4_decompress_*_continue() : These decoding functions allow decompression of multiple blocks in "streaming" mode. Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) In the case of a ring buffers, decoding buffer must be either : @@ -320,50 +306,135 @@ LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecod LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); -/*! -*_usingDict() : -Advanced decoding functions : - These decoding functions work the same as - a combination of LZ4_setStreamDecode() followed by LZ4_decompress_x_continue() - They are stand-alone. They don't need nor update an LZ4_streamDecode_t structure. -*/ +/*! LZ4_decompress_*_usingDict() : + * These decoding functions work the same as + * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() + * They are stand-alone, and don't need an LZ4_streamDecode_t structure. + */ LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); +/*^********************************************** + * !!!!!! STATIC LINKING ONLY !!!!!! + ***********************************************/ +/*-************************************ + * Private definitions + ************************************** + * Do not use these definitions. + * They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. + * Using these definitions will expose code to API and/or ABI break in future versions of the library. + **************************************/ +#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) +#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) +#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ + +#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +#include <stdint.h> + +typedef struct { + uint32_t hashTable[LZ4_HASH_SIZE_U32]; + uint32_t currentOffset; + uint32_t initCheck; + const uint8_t* dictionary; + uint8_t* bufferStart; /* obsolete, used for slideInputBuffer */ + uint32_t dictSize; +} LZ4_stream_t_internal; + +typedef struct { + const uint8_t* externalDict; + size_t extDictSize; + const uint8_t* prefixEnd; + size_t prefixSize; +} LZ4_streamDecode_t_internal; + +#else + +typedef struct { + unsigned int hashTable[LZ4_HASH_SIZE_U32]; + unsigned int currentOffset; + unsigned int initCheck; + const unsigned char* dictionary; + unsigned char* bufferStart; /* obsolete, used for slideInputBuffer */ + unsigned int dictSize; +} LZ4_stream_t_internal; + +typedef struct { + const unsigned char* externalDict; + size_t extDictSize; + const unsigned char* prefixEnd; + size_t prefixSize; +} LZ4_streamDecode_t_internal; + +#endif + +/*! + * LZ4_stream_t : + * information structure to track an LZ4 stream. + * init this structure before first use. + * note : only use in association with static linking ! + * this definition is not API/ABI safe, + * and may change in a future version ! + */ +#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) +#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) +struct LZ4_stream_s { + union { + unsigned long long table[LZ4_STREAMSIZE_U64]; + LZ4_stream_t_internal internal_donotuse; + }; +} ; /* previously typedef'd to LZ4_stream_t */ + + +/*! + * LZ4_streamDecode_t : + * information structure to track an LZ4 stream during decompression. + * init this structure using LZ4_setStreamDecode (or memset()) before first use + * note : only use in association with static linking ! + * this definition is not API/ABI safe, + * and may change in a future version ! + */ +#define LZ4_STREAMDECODESIZE_U64 4 +#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) +struct LZ4_streamDecode_s { + union { + unsigned long long table[LZ4_STREAMDECODESIZE_U64]; + LZ4_streamDecode_t_internal internal_donotuse; + }; +} ; /* previously typedef'd to LZ4_streamDecode_t */ + + /*=************************************ * Obsolete Functions **************************************/ -/* Deprecate Warnings */ -/* Should these warnings messages be a problem, +/* Deprecation warnings */ +/* Should these warnings be a problem, it is generally possible to disable them, - with -Wno-deprecated-declarations for gcc - or _CRT_SECURE_NO_WARNINGS in Visual for example. - Otherwise, you can also define LZ4_DISABLE_DEPRECATE_WARNINGS */ -#define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + typically with -Wno-deprecated-declarations for gcc + or _CRT_SECURE_NO_WARNINGS in Visual. + Otherwise, it's also possible to define LZ4_DISABLE_DEPRECATE_WARNINGS */ #ifdef LZ4_DISABLE_DEPRECATE_WARNINGS -# define LZ4_DEPRECATED() /* disable deprecation warnings */ +# define LZ4_DEPRECATED(message) /* disable deprecation warnings */ #else -# if (LZ4_GCC_VERSION >= 405) || defined(__clang__) +# if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) # define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) -# elif (LZ4_GCC_VERSION >= 301) +# elif defined(__GNUC__) && (__GNUC__ >= 3) # define LZ4_DEPRECATED(message) __attribute__((deprecated)) # elif defined(_MSC_VER) # define LZ4_DEPRECATED(message) __declspec(deprecated(message)) # else -# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") +# warning "WARNING: You need to implement LZ4_DEPRECATED for this compiler" # define LZ4_DEPRECATED(message) # endif #endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ /* Obsolete compression functions */ -/* These functions will generate warnings in a future release */ -LZ4LIB_API int LZ4_compress (const char* source, char* dest, int sourceSize); -LZ4LIB_API int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); -LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); -LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); -LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); -LZ4LIB_API int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress (const char* source, char* dest, int sourceSize); +LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); /* Obsolete decompression functions */ /* These function names are completely deprecated and must no longer be used. diff --git a/lib/lz4frame.c b/lib/lz4frame.c index f5b62dc..c31f82d 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -158,7 +158,7 @@ static void LZ4F_writeLE64 (void* dst, U64 value64) #define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB static const size_t minFHSize = 7; -static const size_t maxFHSize = 15; +static const size_t maxFHSize = 15; /* max Frame Header Size */ static const size_t BHSize = 4; @@ -201,10 +201,16 @@ const char* LZ4F_getErrorName(LZ4F_errorCode_t code) return codeError; } +LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult) +{ + if (!LZ4F_isError(functionResult)) return LZ4F_OK_NoError; + return (LZ4F_errorCodes)(-(ptrdiff_t)functionResult); +} + static LZ4F_errorCode_t err0r(LZ4F_errorCodes code) { LZ4_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t)); /* A compilation error here means sizeof(ptrdiff_t) is not large enough */ - return (LZ4F_errorCode_t)-(ptrdiff_t)code; + return (LZ4F_errorCode_t)-(ptrdiff_t)code; } unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; } @@ -213,6 +219,8 @@ unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; } /*-************************************ * Private functions **************************************/ +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + static size_t LZ4F_getBlockSize(unsigned blockSizeID) { static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB }; @@ -223,7 +231,6 @@ static size_t LZ4F_getBlockSize(unsigned blockSizeID) return blockSizes[blockSizeID]; } - static BYTE LZ4F_headerChecksum (const void* header, size_t length) { U32 const xxh = XXH32(header, length, 0); @@ -232,14 +239,13 @@ static BYTE LZ4F_headerChecksum (const void* header, size_t length) /*-************************************ -* Simple compression functions +* Simple-pass compression functions **************************************/ static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID, const size_t srcSize) { LZ4F_blockSizeID_t proposedBSID = LZ4F_max64KB; size_t maxBlockSize = 64 KB; - while (requestedBSID > proposedBSID) - { + while (requestedBSID > proposedBSID) { if (srcSize <= maxBlockSize) return proposedBSID; proposedBSID = (LZ4F_blockSizeID_t)((int)proposedBSID + 1); @@ -248,23 +254,39 @@ static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSI return requestedBSID; } +static size_t LZ4F_compressBound_internal(size_t srcSize, const LZ4F_preferences_t* preferencesPtr, size_t alreadyBuffered) +{ + LZ4F_preferences_t prefsNull; + memset(&prefsNull, 0, sizeof(prefsNull)); + prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */ + { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; + LZ4F_blockSizeID_t const bid = prefsPtr->frameInfo.blockSizeID; + size_t const blockSize = LZ4F_getBlockSize(bid); + size_t const maxBuffered = blockSize - 1; + size_t const bufferedSize = MIN(alreadyBuffered, maxBuffered); + size_t const maxSrcSize = srcSize + bufferedSize; + unsigned const nbFullBlocks = (unsigned)(maxSrcSize / blockSize); + size_t const partialBlockSize = srcSize & (blockSize-1); + size_t const lastBlockSize = prefsPtr->autoFlush ? partialBlockSize : 0; + unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0); + + size_t const blockHeaderSize = 4; /* default, without block CRC option (which cannot be generated with current API) */ + size_t const frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); + + return (blockHeaderSize * nbBlocks) + (blockSize * nbFullBlocks) + lastBlockSize + frameEnd;; + } +} size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { LZ4F_preferences_t prefs; - size_t headerSize; - size_t streamSize; + size_t const headerSize = maxFHSize; /* max header size, including magic number and frame content size */ if (preferencesPtr!=NULL) prefs = *preferencesPtr; else memset(&prefs, 0, sizeof(prefs)); - - prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); prefs.autoFlush = 1; - headerSize = maxFHSize; /* header size, including magic number and frame content size*/ - streamSize = LZ4F_compressBound(srcSize, &prefs); - - return headerSize + streamSize; + return headerSize + LZ4F_compressBound_internal(srcSize, &prefs, 0);; } @@ -376,11 +398,11 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp /*! LZ4F_compressBegin() : * will write the frame header into dstBuffer. - * dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is LZ4F_MAXHEADERFRAME_SIZE bytes. + * dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_MAXHEADERFRAME_SIZE bytes. * @return : number of bytes written into dstBuffer for the header * or an error code (can be tested using LZ4F_isError()) */ -size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, const LZ4F_preferences_t* preferencesPtr) +size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* preferencesPtr) { LZ4F_preferences_t prefNull; BYTE* const dstStart = (BYTE*)dstBuffer; @@ -388,7 +410,7 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize BYTE* headerStart; size_t requiredBuffSize; - if (dstMaxSize < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); + if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); if (cctxPtr->cStage != 0) return err0r(LZ4F_ERROR_GENERIC); memset(&prefNull, 0, sizeof(prefNull)); if (preferencesPtr == NULL) preferencesPtr = &prefNull; @@ -456,25 +478,14 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize } -/* LZ4F_compressBound() : gives the size of Dst buffer given a srcSize to handle worst case situations. -* The LZ4F_frameInfo_t structure is optional : -* you can provide NULL as argument, preferences will then be set to cover worst case situations. -* */ +/* LZ4F_compressBound() : + * @ return size of Dst buffer given a srcSize to handle worst case situations. + * The LZ4F_frameInfo_t structure is optional : if NULL, preferences will be set to cover worst case situations. + * This function cannot fail. + */ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { - LZ4F_preferences_t prefsNull; - memset(&prefsNull, 0, sizeof(prefsNull)); - prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */ - { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; - LZ4F_blockSizeID_t const bid = prefsPtr->frameInfo.blockSizeID; - size_t const blockSize = LZ4F_getBlockSize(bid); - unsigned const nbBlocks = (unsigned)(srcSize / blockSize) + 1; - size_t const lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize; - size_t const blockInfo = 4; /* default, without block CRC option */ - size_t const frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); - - return (blockInfo * nbBlocks) + (blockSize * (nbBlocks-1)) + lastBlockSize + frameEnd;; - } + return LZ4F_compressBound_internal(srcSize, preferencesPtr, (size_t)-1); } @@ -482,29 +493,29 @@ typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx, int level) { - /* compress one block */ + /* compress a single block */ BYTE* const cSizePtr = (BYTE*)dst; U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1), level); LZ4F_writeLE32(cSizePtr, cSize); if (cSize == 0) { /* compression failed */ cSize = (U32)srcSize; - LZ4F_writeLE32(cSizePtr, cSize + LZ4F_BLOCKUNCOMPRESSED_FLAG); + LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG); memcpy(cSizePtr+4, src, srcSize); } return cSize + 4; } -static int LZ4F_localLZ4_compress_limitedOutput_withState(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) +static int LZ4F_localLZ4_compress_limitedOutput_withState(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level) { (void) level; - return LZ4_compress_limitedOutput_withState(ctx, src, dst, srcSize, dstSize); + return LZ4_compress_fast_extState(ctx, src, dst, srcSize, dstCapacity, 1); } -static int LZ4F_localLZ4_compress_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) +static int LZ4F_localLZ4_compress_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level) { (void) level; - return LZ4_compress_limitedOutput_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstSize); + return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, 1); } static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) @@ -534,14 +545,14 @@ typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus; /*! LZ4F_compressUpdate() : * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. -* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. +* The most important rule is that dstBuffer MUST be large enough (dstCapacity) to ensure compression completion even in worst case. * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode) -* You can get the minimum value of dstMaxSize by using LZ4F_compressBound() +* You can get the minimum value of dstCapacity by using LZ4F_compressBound() * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. * The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered. * The function outputs an error code if it fails (can be tested using LZ4F_isError()) */ -size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr) +size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr) { LZ4F_compressOptions_t cOptionsNull; size_t const blockSize = cctxPtr->maxBlockSize; @@ -550,17 +561,14 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSiz BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; LZ4F_lastBlockStatus lastBlockCompressed = notDone; - compressFunc_t compress; + compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC); - if (dstMaxSize < LZ4F_compressBound(srcSize, &(cctxPtr->prefs))) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); + if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize)) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); memset(&cOptionsNull, 0, sizeof(cOptionsNull)); if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; - /* select compression function */ - compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); - /* complete tmp buffer */ if (cctxPtr->tmpInSize > 0) { /* some data already within tmp buffer */ size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize; @@ -640,7 +648,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSiz * The function outputs an error code if it fails (can be tested using LZ4F_isError()) * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. */ -size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) +size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* compressOptionsPtr) { BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; @@ -648,7 +656,7 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, const if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */ if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC); - if (dstMaxSize < (cctxPtr->tmpInSize + 8)) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); /* +8 : block header(4) + block checksum(4) */ + if (dstCapacity < (cctxPtr->tmpInSize + 4)) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); /* +4 : block header(4) */ (void)compressOptionsPtr; /* not yet useful */ /* select compression function */ diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 8fc5d17..e26be76 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -151,7 +151,8 @@ typedef struct { * Simple compression function ***********************************/ /*!LZ4F_compressFrameBound() : - * Returns the maximum possible size of a frame given srcSize content and preferences. + * Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences. + * Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression. */ LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr); @@ -175,7 +176,7 @@ typedef struct LZ4F_cctx_s LZ4F_cctx; /* incomplete type */ typedef LZ4F_cctx* LZ4F_compressionContext_t; /* for compatibility with previous API version */ typedef struct { - unsigned stableSrc; /* 1 == src content remain present on future calls to LZ4F_compress(); avoid saving src content within tmp buffer as future dictionary */ + unsigned stableSrc; /* 1 == src content will remain present on future calls to LZ4F_compress(); skip copying src content within tmp buffer */ unsigned reserved[3]; } LZ4F_compressOptions_t; diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h index 0c154a3..d922e39 100644 --- a/lib/lz4frame_static.h +++ b/lib/lz4frame_static.h @@ -72,6 +72,8 @@ extern "C" { #endif typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; /* enum is exposed, to handle specific errors; compare function result to -enum value */ +LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult); + #if defined (__cplusplus) } diff --git a/lib/lz4hc.c b/lib/lz4hc.c index edc2db0..6de8e8f 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -31,7 +31,7 @@ - LZ4 source repository : https://github.com/lz4/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ - +/* note : lz4hc is not an independent module, it requires lz4.h/lz4.c for proper compilation */ /* ************************************* @@ -44,11 +44,13 @@ * in stack (0:fastest), or in heap (1:requires malloc()). * Since workplace is rather large, heap mode is recommended. */ -#define LZ4HC_HEAPMODE 0 +#ifndef LZ4HC_HEAPMODE +# define LZ4HC_HEAPMODE 1 +#endif /* ************************************* -* Includes +* Dependency ***************************************/ #include "lz4hc.h" @@ -75,40 +77,14 @@ /* ************************************* * Local Constants ***************************************/ -#define DICTIONARY_LOGSIZE 16 -#define MAXD (1<<DICTIONARY_LOGSIZE) -#define MAXD_MASK (MAXD - 1) - -#define HASH_LOG (DICTIONARY_LOGSIZE-1) -#define HASHTABLESIZE (1 << HASH_LOG) -#define HASH_MASK (HASHTABLESIZE - 1) - #define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH) /************************************** -* Local Types -**************************************/ -typedef struct -{ - U32 hashTable[HASHTABLESIZE]; - U16 chainTable[MAXD]; - const BYTE* end; /* next block here to continue on current prefix */ - const BYTE* base; /* All index relative to this position */ - const BYTE* dictBase; /* alternate base for extDict */ - BYTE* inputBuffer; /* deprecated */ - U32 dictLimit; /* below that point, need extDict */ - U32 lowLimit; /* below that point, no more dict */ - U32 nextToUpdate; /* index from which to continue dictionary update */ - U32 compressionLevel; -} LZ4HC_Data_Structure; - - -/************************************** * Local Macros **************************************/ -#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG)) -/* #define DELTANEXTU16(p) chainTable[(p) & MAXD_MASK] */ /* flexible, MAXD dependent */ +#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG)) +/* #define DELTANEXTU16(p) chainTable[(p) & LZ4HC_MAXD_MASK] */ /* flexible, LZ4HC_MAXD dependent */ #define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } @@ -118,7 +94,7 @@ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr) /************************************** * HC Compression **************************************/ -static void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* start) +static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) { MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); @@ -132,7 +108,7 @@ static void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* start) /* Update chains up to ip (excluded) */ -FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip) +FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip) { U16* const chainTable = hc4->chainTable; U32* const hashTable = hc4->hashTable; @@ -153,7 +129,7 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip) } -FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, /* Index table will be updated */ +FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* hc4, /* Index table will be updated */ const BYTE* ip, const BYTE* const iLimit, const BYTE** matchpos, const int maxNbAttempts) @@ -165,7 +141,6 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, /* I const U32 dictLimit = hc4->dictLimit; const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); U32 matchIndex; - const BYTE* match; int nbAttempts=maxNbAttempts; size_t ml=0; @@ -176,7 +151,7 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, /* I while ((matchIndex>=lowLimit) && (nbAttempts)) { nbAttempts--; if (matchIndex >= dictLimit) { - match = base + matchIndex; + const BYTE* const match = base + matchIndex; if (*(match+ml) == *(ip+ml) && (LZ4_read32(match) == LZ4_read32(ip))) { @@ -184,7 +159,7 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, /* I if (mlt > ml) { ml = mlt; *matchpos = match; } } } else { - match = dictBase + matchIndex; + const BYTE* const match = dictBase + matchIndex; if (LZ4_read32(match) == LZ4_read32(ip)) { size_t mlt; const BYTE* vLimit = ip + (dictLimit - matchIndex); @@ -203,7 +178,7 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, /* I FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( - LZ4HC_Data_Structure* hc4, + LZ4HC_CCtx_internal* hc4, const BYTE* const ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit, @@ -332,7 +307,7 @@ FORCE_INLINE int LZ4HC_encodeSequence ( static int LZ4HC_compress_generic ( - void* const ctxvoid, + LZ4HC_CCtx_internal* const ctx, const char* const source, char* const dest, int const inputSize, @@ -341,7 +316,6 @@ static int LZ4HC_compress_generic ( limitedOutput_directive limit ) { - LZ4HC_Data_Structure* ctx = (LZ4HC_Data_Structure*) ctxvoid; const BYTE* ip = (const BYTE*) source; const BYTE* anchor = ip; const BYTE* const iend = ip + inputSize; @@ -516,28 +490,29 @@ _Search3: } -int LZ4_sizeofStateHC(void) { return sizeof(LZ4HC_Data_Structure); } +int LZ4_sizeofStateHC(void) { return sizeof(LZ4_streamHC_t); } int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel) { + LZ4HC_CCtx_internal* ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ - LZ4HC_init ((LZ4HC_Data_Structure*)state, (const BYTE*)src); + LZ4HC_init (ctx, (const BYTE*)src); if (maxDstSize < LZ4_compressBound(srcSize)) - return LZ4HC_compress_generic (state, src, dst, srcSize, maxDstSize, compressionLevel, limitedOutput); + return LZ4HC_compress_generic (ctx, src, dst, srcSize, maxDstSize, compressionLevel, limitedOutput); else - return LZ4HC_compress_generic (state, src, dst, srcSize, maxDstSize, compressionLevel, noLimit); + return LZ4HC_compress_generic (ctx, src, dst, srcSize, maxDstSize, compressionLevel, noLimit); } int LZ4_compress_HC(const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel) { -#if LZ4HC_HEAPMODE==1 - LZ4HC_Data_Structure* statePtr = malloc(sizeof(LZ4HC_Data_Structure)); +#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 + LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); #else - LZ4HC_Data_Structure state; - LZ4HC_Data_Structure* const statePtr = &state; + LZ4_streamHC_t state; + LZ4_streamHC_t* const statePtr = &state; #endif - int cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, maxDstSize, compressionLevel); -#if LZ4HC_HEAPMODE==1 + int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, maxDstSize, compressionLevel); +#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 free(statePtr); #endif return cSize; @@ -556,14 +531,14 @@ int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { free(LZ4_st /* initialization */ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) { - LZ4_STATIC_ASSERT(sizeof(LZ4HC_Data_Structure) <= sizeof(LZ4_streamHC_t)); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ - ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base = NULL; - ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel = (unsigned)compressionLevel; + LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ + LZ4_streamHCPtr->internal_donotuse.base = NULL; + LZ4_streamHCPtr->internal_donotuse.compressionLevel = (unsigned)compressionLevel; } int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) { - LZ4HC_Data_Structure* ctxPtr = (LZ4HC_Data_Structure*) LZ4_streamHCPtr; + LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse; if (dictSize > 64 KB) { dictionary += dictSize - 64 KB; dictSize = 64 KB; @@ -577,7 +552,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int /* compression */ -static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock) +static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) { if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ @@ -589,10 +564,11 @@ static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newB ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ } -static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* ctxPtr, +static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize, limitedOutput_directive limit) { + LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse; /* auto-init if forgotten */ if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) source); @@ -600,7 +576,7 @@ static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* ctxPtr, if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) { size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit; if (dictSize > 64 KB) dictSize = 64 KB; - LZ4_loadDictHC((LZ4_streamHC_t*)ctxPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize); + LZ4_loadDictHC(LZ4_streamHCPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize); } /* Check if blocks follow each other */ @@ -623,9 +599,9 @@ static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* ctxPtr, int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize) { if (maxOutputSize < LZ4_compressBound(inputSize)) - return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput); + return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput); else - return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, noLimit); + return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, noLimit); } @@ -633,7 +609,7 @@ int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* sourc int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize) { - LZ4HC_Data_Structure* const streamPtr = (LZ4HC_Data_Structure*)LZ4_streamHCPtr; + LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse; int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit)); if (dictSize > 64 KB) dictSize = 64 KB; if (dictSize < 4) dictSize = 0; @@ -653,8 +629,8 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS /*********************************** * Deprecated Functions ***********************************/ +/* These functions currently generate deprecation warnings */ /* Deprecated compression functions */ -/* These functions are planned to start generate warnings by r131 approximately */ int LZ4_compressHC(const char* src, char* dst, int srcSize) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), 0); } int LZ4_compressHC_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, 0); } int LZ4_compressHC2(const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); } @@ -668,45 +644,41 @@ int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* ctx, const char* src, /* Deprecated streaming functions */ -/* These functions currently generate deprecation warnings */ int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; } int LZ4_resetStreamStateHC(void* state, char* inputBuffer) { + LZ4HC_CCtx_internal *ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */ - LZ4HC_init((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer); - ((LZ4HC_Data_Structure*)state)->inputBuffer = (BYTE*)inputBuffer; + LZ4HC_init(ctx, (const BYTE*)inputBuffer); + ctx->inputBuffer = (BYTE*)inputBuffer; return 0; } void* LZ4_createHC (char* inputBuffer) { - void* hc4 = ALLOCATOR(1, sizeof(LZ4HC_Data_Structure)); + LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOCATOR(1, sizeof(LZ4_streamHC_t)); if (hc4 == NULL) return NULL; /* not enough memory */ - LZ4HC_init ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer); - ((LZ4HC_Data_Structure*)hc4)->inputBuffer = (BYTE*)inputBuffer; + LZ4HC_init (&hc4->internal_donotuse, (const BYTE*)inputBuffer); + hc4->internal_donotuse.inputBuffer = (BYTE*)inputBuffer; return hc4; } -int LZ4_freeHC (void* LZ4HC_Data) -{ - FREEMEM(LZ4HC_Data); - return (0); -} +int LZ4_freeHC (void* LZ4HC_Data) { FREEMEM(LZ4HC_Data); return 0; } int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel) { - return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, compressionLevel, noLimit); + return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, source, dest, inputSize, 0, compressionLevel, noLimit); } int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel) { - return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput); + return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput); } char* LZ4_slideInputBufferHC(void* LZ4HC_Data) { - LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data; - int dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB); + LZ4HC_CCtx_internal* const hc4 = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse; + int const dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB); return (char*)(hc4->inputBuffer + dictSize); } diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 563ad98..7adc8fa 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -1,7 +1,7 @@ /* LZ4 HC - High Compression Mode of LZ4 Header File - Copyright (C) 2011-2015, Yann Collet. + Copyright (C) 2011-2016, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -34,121 +34,85 @@ #ifndef LZ4_HC_H_19834876238432 #define LZ4_HC_H_19834876238432 - #if defined (__cplusplus) extern "C" { #endif -/*-*************************** -* Includes -*****************************/ -#include <stddef.h> /* size_t */ - -/*-*************************************************************** -* Export parameters -*****************************************************************/ -/*! -* LZ4_DLL_EXPORT : -* Enable exporting of functions when building a Windows DLL -*/ -#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) -# define LZ4HCLIB_API __declspec(dllexport) -#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) -# define LZ4HCLIB_API __declspec(dllimport) -#else -# define LZ4HCLIB_API -#endif +/* --- Dependency --- */ +/* note : lz4hc is not an independent module, it requires lz4.h/lz4.c for proper compilation */ +#include "lz4.h" /* stddef, LZ4LIB_API, LZ4_DEPRECATED */ +/* --- Useful constants --- */ #define LZ4HC_MIN_CLEVEL 3 #define LZ4HC_DEFAULT_CLEVEL 9 #define LZ4HC_MAX_CLEVEL 16 + /*-************************************ -* Block Compression -**************************************/ -/*! -LZ4_compress_HC() : - Destination buffer `dst` must be already allocated. - Compression success is guaranteed if `dst` buffer is sized to handle worst circumstances (data not compressible) - Worst size evaluation is provided by function LZ4_compressBound() (see "lz4.h") - `srcSize` : Max supported value is LZ4_MAX_INPUT_SIZE (see "lz4.h") - `compressionLevel` : Recommended values are between 4 and 9, although any value between 0 and LZ4HC_MAX_CLEVEL will work. - 0 means "use default value" (see lz4hc.c). - Values >LZ4HC_MAX_CLEVEL behave the same as 16. - @return : the number of bytes written into buffer 'dst' - or 0 if compression fails. -*/ -LZ4HCLIB_API int LZ4_compress_HC (const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); + * Block Compression + **************************************/ +/*! LZ4_compress_HC() : + * Compress data from `src` into `dst`, using the more powerful but slower "HC" algorithm. + * `dst` must be already allocated. + * Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h") + * Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h") + * `compressionLevel` : Recommended values are between 4 and 9, although any value between 1 and LZ4HC_MAX_CLEVEL will work. + * Values >LZ4HC_MAX_CLEVEL behave the same as 16. + * @return : the number of bytes written into 'dst' + * or 0 if compression fails. + */ +LZ4LIB_API int LZ4_compress_HC (const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel); /* Note : - Decompression functions are provided within "lz4.h" (BSD license) -*/ + * Decompression functions are provided within "lz4.h" (BSD license) + */ -/*! -LZ4_compress_HC_extStateHC() : - Use this function if you prefer to manually allocate memory for compression tables. - To know how much memory must be allocated for the compression tables, use : - int LZ4_sizeofStateHC(); - - Allocated memory must be aligned on 8-bytes boundaries (which a normal malloc() will do properly). - - The allocated memory can then be provided to the compression functions using 'void* state' parameter. - LZ4_compress_HC_extStateHC() is equivalent to previously described function. - It just uses externally allocated memory for stateHC. -*/ -LZ4HCLIB_API int LZ4_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); -LZ4HCLIB_API int LZ4_sizeofStateHC(void); +/*! LZ4_compress_HC_extStateHC() : + * Same as LZ4_compress_HC(), but using an externally allocated memory segment for `state`. + * `state` size is provided by LZ4_sizeofStateHC(). + * Memory segment must be aligned on 8-bytes boundaries (which a normal malloc() will do properly). + */ +LZ4LIB_API int LZ4_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); +LZ4LIB_API int LZ4_sizeofStateHC(void); /*-************************************ -* Streaming Compression -**************************************/ -#define LZ4_STREAMHCSIZE 262192 -#define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t)) -typedef struct { size_t table[LZ4_STREAMHCSIZE_SIZET]; } LZ4_streamHC_t; -/* - LZ4_streamHC_t - This structure allows static allocation of LZ4 HC streaming state. - State must then be initialized using LZ4_resetStreamHC() before first use. - - Static allocation should only be used in combination with static linking. - If you want to use LZ4 as a DLL, please use construction functions below, which are future-proof. -*/ - + * Streaming Compression + * Bufferless synchronous API + **************************************/ + typedef struct LZ4_streamHC_s LZ4_streamHC_t; /* incomplete type (defined later) */ /*! LZ4_createStreamHC() and LZ4_freeStreamHC() : - These functions create and release memory for LZ4 HC streaming state. - Newly created states are already initialized. - Existing state space can be re-used anytime using LZ4_resetStreamHC(). - If you use LZ4 as a DLL, use these functions instead of static structure allocation, - to avoid size mismatch between different versions. -*/ -LZ4HCLIB_API LZ4_streamHC_t* LZ4_createStreamHC(void); -LZ4HCLIB_API int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr); + * These functions create and release memory for LZ4 HC streaming state. + * Newly created states are automatically initialized. + * Existing states can be re-used several times, using LZ4_resetStreamHC(). + * These methods are API and ABI stable, they can be used in combination with a DLL. + */ +LZ4LIB_API LZ4_streamHC_t* LZ4_createStreamHC(void); +LZ4LIB_API int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr); -LZ4HCLIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionLevel); -LZ4HCLIB_API int LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize); +LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionLevel); +LZ4LIB_API int LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize); -LZ4HCLIB_API int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, const char* src, char* dst, int srcSize, int maxDstSize); +LZ4LIB_API int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, const char* src, char* dst, int srcSize, int maxDstSize); -LZ4HCLIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize); +LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize); /* These functions compress data in successive blocks of any size, using previous blocks as dictionary. One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks. There is an exception for ring buffers, which can be smaller than 64 KB. - Such case is automatically detected and correctly handled by LZ4_compress_HC_continue(). + Ring buffers scenario is automatically detected and handled by LZ4_compress_HC_continue(). Before starting compression, state must be properly initialized, using LZ4_resetStreamHC(). A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional). Then, use LZ4_compress_HC_continue() to compress each successive block. - It works like LZ4_compress_HC(), but use previous memory blocks as dictionary to improve compression. Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression. - As a reminder, size 'dst' buffer to handle worst cases, using LZ4_compressBound(), to ensure success of compression operation. + 'dst' buffer should be sized to handle worst case scenarios, using LZ4_compressBound(), to ensure operation success. If, for any reason, previous data blocks can't be preserved unmodified in memory during next compression block, you must save it to a safer memory space, using LZ4_saveDictHC(). @@ -156,50 +120,102 @@ LZ4HCLIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, */ +/*-****************************************** + * !!!!! STATIC LINKING ONLY !!!!! + *******************************************/ + + /*-************************************* + * PRIVATE DEFINITIONS : + * Do not use these definitions. + * They are exposed to allow static allocation of `LZ4_streamHC_t`. + * Using these definitions makes the code vulnerable to potential API break when upgrading LZ4 + **************************************/ +#define LZ4HC_DICTIONARY_LOGSIZE 16 +#define LZ4HC_MAXD (1<<LZ4HC_DICTIONARY_LOGSIZE) +#define LZ4HC_MAXD_MASK (LZ4HC_MAXD - 1) + +#define LZ4HC_HASH_LOG (LZ4HC_DICTIONARY_LOGSIZE-1) +#define LZ4HC_HASHTABLESIZE (1 << LZ4HC_HASH_LOG) +#define LZ4HC_HASH_MASK (LZ4HC_HASHTABLESIZE - 1) + + +#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +#include <stdint.h> + +typedef struct +{ + uint32_t hashTable[LZ4HC_HASHTABLESIZE]; + uint16_t chainTable[LZ4HC_MAXD]; + const uint8_t* end; /* next block here to continue on current prefix */ + const uint8_t* base; /* All index relative to this position */ + const uint8_t* dictBase; /* alternate base for extDict */ + uint8_t* inputBuffer; /* deprecated */ + uint32_t dictLimit; /* below that point, need extDict */ + uint32_t lowLimit; /* below that point, no more dict */ + uint32_t nextToUpdate; /* index from which to continue dictionary update */ + uint32_t compressionLevel; +} LZ4HC_CCtx_internal; + +#else + +typedef struct +{ + unsigned int hashTable[LZ4HC_HASHTABLESIZE]; + unsigned short chainTable[LZ4HC_MAXD]; + const unsigned char* end; /* next block here to continue on current prefix */ + const unsigned char* base; /* All index relative to this position */ + const unsigned char* dictBase; /* alternate base for extDict */ + unsigned char* inputBuffer; /* deprecated */ + unsigned int dictLimit; /* below that point, need extDict */ + unsigned int lowLimit; /* below that point, no more dict */ + unsigned int nextToUpdate; /* index from which to continue dictionary update */ + unsigned int compressionLevel; +} LZ4HC_CCtx_internal; + +#endif + +#define LZ4_STREAMHCSIZE 262192 +#define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t)) +struct LZ4_streamHC_s { + union { + size_t table[LZ4_STREAMHCSIZE_SIZET]; + LZ4HC_CCtx_internal internal_donotuse; + }; +}; /* previously typedef'd to LZ4_streamHC_t */ +/* + LZ4_streamHC_t : + This structure allows static allocation of LZ4 HC streaming state. + State must be initialized using LZ4_resetStreamHC() before first use. + + Static allocation shall only be used in combination with static linking. + When invoking LZ4 from a DLL, use create/free functions instead, which are API and ABI stable. +*/ + /*-************************************ * Deprecated Functions **************************************/ -/* Deprecate Warnings */ -/* Should these warnings messages be a problem, - it is generally possible to disable them, - with -Wno-deprecated-declarations for gcc - or _CRT_SECURE_NO_WARNINGS in Visual for example. - You can also define LZ4_DEPRECATE_WARNING_DEFBLOCK. */ -#ifndef LZ4_DEPRECATE_WARNING_DEFBLOCK -# define LZ4_DEPRECATE_WARNING_DEFBLOCK -# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# if (LZ4_GCC_VERSION >= 405) || defined(__clang__) -# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) -# elif (LZ4_GCC_VERSION >= 301) -# define LZ4_DEPRECATED(message) __attribute__((deprecated)) -# elif defined(_MSC_VER) -# define LZ4_DEPRECATED(message) __declspec(deprecated(message)) -# else -# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") -# define LZ4_DEPRECATED(message) -# endif -#endif /* LZ4_DEPRECATE_WARNING_DEFBLOCK */ +/* see lz4.h LZ4_DISABLE_DEPRECATE_WARNINGS to turn off deprecation warnings */ /* deprecated compression functions */ /* these functions will trigger warning messages in future releases */ -LZ4HCLIB_API int LZ4_compressHC (const char* source, char* dest, int inputSize); -LZ4HCLIB_API int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC (const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel); LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); -LZ4HCLIB_API int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); -LZ4HCLIB_API int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel); LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); -LZ4HCLIB_API int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize); -LZ4HCLIB_API int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); /* Deprecated Streaming functions using older model; should no longer be used */ LZ4_DEPRECATED("use LZ4_createStreamHC() instead") void* LZ4_createHC (char* inputBuffer); LZ4_DEPRECATED("use LZ4_saveDictHC() instead") char* LZ4_slideInputBufferHC (void* LZ4HC_Data); LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") int LZ4_freeHC (void* LZ4HC_Data); -LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); -LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); +LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); +LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); LZ4_DEPRECATED("use LZ4_createStreamHC() instead") int LZ4_sizeofStreamStateHC(void); LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStreamStateHC(void* state, char* inputBuffer); diff --git a/programs/lz4.1 b/programs/lz4.1 index 1d9238d..434b131 100644 --- a/programs/lz4.1 +++ b/programs/lz4.1 @@ -217,7 +217,7 @@ hence for a file. It won't work with unknown source size, such as stdin or pipe. .BR \-q ", " --quiet suppress warnings and real-time statistics; specify twice to suppress errors too .TP -.B \-h/\-H +.B \-h/\-H ", " --help display help/long help and exit .TP .BR \-V ", " \--version diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 1f25a21..88fbb53 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -351,6 +351,7 @@ int main(int argc, const char** argv) if (!strcmp(argument, "--verbose")) { displayLevel++; continue; } if (!strcmp(argument, "--quiet")) { if (displayLevel) displayLevel--; continue; } if (!strcmp(argument, "--version")) { DISPLAY(WELCOME_MESSAGE); return 0; } + if (!strcmp(argument, "--help")) { usage_advanced(exeName); goto _cleanup; } if (!strcmp(argument, "--keep")) { LZ4IO_setRemoveSrcFile(0); continue; } /* keep source file (default) */ if (!strcmp(argument, "--rm")) { LZ4IO_setRemoveSrcFile(1); continue; } } diff --git a/tests/Makefile b/tests/Makefile index 2b88f73..4481392 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -66,7 +66,7 @@ default: bins m32: fullbench32 fuzzer32 frametest32 -bins: fullbench fuzzer frametest datagen +bins: fullbench fuzzer frametest datagen fasttest all: bins m32 diff --git a/tests/fasttest.c b/tests/fasttest.c index a405542..7694e0c 100644 --- a/tests/fasttest.c +++ b/tests/fasttest.c @@ -36,8 +36,8 @@ int test_compress(const char *input, int inSize, char *output, int outSize) if (inSize > 1024) return -2; if (outSize - (outOffset + 8) < LZ4_compressBound(length)) return -3; { - const int outBytes = LZ4_compress_continue( - lz4Stream, input + inOffset, output + outOffset + 8, length); + const int outBytes = LZ4_compress_fast_continue( + lz4Stream, input + inOffset, output + outOffset + 8, length, outSize-outOffset, 1); if(outBytes <= 0) return -4; memcpy(output + outOffset, &length, 4); /* input length */ memcpy(output + outOffset + 4, &outBytes, 4); /* output length */ diff --git a/tests/frametest.c b/tests/frametest.c index ccf4278..d4afc58 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -48,7 +48,7 @@ /*-************************************ * Basic Types **************************************/ -#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ +#if !defined(__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) # include <stdint.h> typedef uint8_t BYTE; typedef uint16_t U16; @@ -109,7 +109,6 @@ static clock_t g_clockTime = 0; * Local Parameters *****************************************/ static U32 no_prompt = 0; -static char* programName; static U32 displayLevel = 2; static U32 pause = 0; @@ -117,6 +116,9 @@ static U32 pause = 0; /*-******************************************************* * Fuzzer functions *********************************************************/ +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +#define MAX(a,b) ( (a) > (b) ? (a) : (b) ) + static clock_t FUZ_GetClockSpan(clock_t clockStart) { return clock() - clockStart; /* works even if overflow; max span ~ 30 mn */ @@ -179,24 +181,28 @@ static unsigned FUZ_highbit(U32 v32) } +/*-******************************************************* +* Tests +*********************************************************/ int basicTests(U32 seed, double compressibility) { int testResult = 0; - void* CNBuffer; - void* compressedBuffer; - void* decodedBuffer; + void* const CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); + size_t const cBuffSize = LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL); + void* const compressedBuffer = malloc(cBuffSize); + void* const decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); U32 randState = seed; size_t cSize, testSize; - LZ4F_preferences_t prefs; LZ4F_decompressionContext_t dCtx = NULL; LZ4F_compressionContext_t cctx = NULL; U64 crcOrig; - /* Create compressible test buffer */ + LZ4F_preferences_t prefs; memset(&prefs, 0, sizeof(prefs)); - CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); - compressedBuffer = malloc(LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL)); - decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); + if (!CNBuffer || !compressedBuffer || !decodedBuffer) { + DISPLAY("allocation error, not enough memory to start fuzzer tests \n"); + goto _output_error; + } FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); @@ -221,49 +227,58 @@ int basicTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "LZ4F_freeDecompressionContext \n"); { LZ4F_errorCode_t const errorCode = LZ4F_freeDecompressionContext(dCtx); if (LZ4F_isError(errorCode)) goto _output_error; } + dCtx = NULL; - /* Trivial tests : one-step frame */ + /* test one-pass frame compression */ testSize = COMPRESSIBLE_NOISE_LENGTH; DISPLAYLEVEL(3, "LZ4F_compressFrame, using default preferences : \n"); cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL); if (LZ4F_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); + DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize); DISPLAYLEVEL(3, "Decompression test : \n"); { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; size_t compressedBufferSize = cSize; - BYTE* op = (BYTE*)decodedBuffer; - BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; BYTE* ip = (BYTE*)compressedBuffer; BYTE* const iend = (BYTE*)compressedBuffer + cSize; - U64 crcDest; LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); if (LZ4F_isError(errorCode)) goto _output_error; - DISPLAYLEVEL(3, "Single Block : \n"); - errorCode = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; - crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); - if (crcDest != crcOrig) goto _output_error; - DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize); + DISPLAYLEVEL(3, "Single Pass decompression : \n"); + { size_t const decompressError = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL); + if (LZ4F_isError(decompressError)) goto _output_error; } + { U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); + if (crcDest != crcOrig) goto _output_error; } + DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedBufferSize); DISPLAYLEVEL(3, "Reusing decompression context \n"); - { size_t iSize = compressedBufferSize - 4; + { size_t const missingBytes = 4; + size_t iSize = compressedBufferSize - missingBytes; const BYTE* cBuff = (const BYTE*) compressedBuffer; - size_t decResult; - DISPLAYLEVEL(3, "Missing last 4 bytes : "); - decResult = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, cBuff, &iSize, NULL); + BYTE* const ostart = (BYTE*)decodedBuffer; + BYTE* op = ostart; + BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; + size_t decResult, oSize = COMPRESSIBLE_NOISE_LENGTH; + DISPLAYLEVEL(3, "Missing last %u bytes : ", (U32)missingBytes); + decResult = LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL); if (LZ4F_isError(decResult)) goto _output_error; - if (!decResult) goto _output_error; /* not finished */ - DISPLAYLEVEL(3, "indeed, request %u bytes \n", (unsigned)decResult); + if (decResult != missingBytes) { + DISPLAY("%u bytes missing != %u bytes requested \n", (U32)missingBytes, (U32)decResult); + goto _output_error; + } + DISPLAYLEVEL(3, "indeed, requests %u bytes \n", (unsigned)decResult); cBuff += iSize; iSize = decResult; - decResult = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, cBuff, &iSize, NULL); + op += oSize; + oSize = (size_t)(oend-op); + decResult = LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL); if (decResult != 0) goto _output_error; /* should finish now */ - crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); - if (crcDest != crcOrig) goto _output_error; - } + op += oSize; + if (op>oend) { DISPLAY("decompression write overflow \n"); goto _output_error; } + { U64 const crcDest = XXH64(decodedBuffer, op-ostart, 1); + if (crcDest != crcOrig) goto _output_error; + } } { size_t oSize = 0; size_t iSize = 0; @@ -295,20 +310,25 @@ int basicTests(U32 seed, double compressibility) } DISPLAYLEVEL(3, "Byte after byte : \n"); - while (ip < iend) { - size_t oSize = oend-op; - size_t iSize = 1; - errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; - op += oSize; - ip += iSize; - } - crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); - if (crcDest != crcOrig) goto _output_error; - DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-(BYTE*)decodedBuffer), COMPRESSIBLE_NOISE_LENGTH); + { BYTE* const ostart = (BYTE*)decodedBuffer; + BYTE* op = ostart; + BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; + while (ip < iend) { + size_t oSize = oend-op; + size_t iSize = 1; + errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); + if (LZ4F_isError(errorCode)) goto _output_error; + op += oSize; + ip += iSize; + } + { U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); + if (crcDest != crcOrig) goto _output_error; } + DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-ostart), COMPRESSIBLE_NOISE_LENGTH); + } errorCode = LZ4F_freeDecompressionContext(dCtx); if (LZ4F_isError(errorCode)) goto _output_error; + dCtx = NULL; } DISPLAYLEVEL(3, "Using 64 KB block : \n"); @@ -332,35 +352,37 @@ int basicTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "Decompression test : \n"); - { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; - unsigned maxBits = FUZ_highbit((U32)decodedBufferSize); - BYTE* op = (BYTE*)decodedBuffer; - BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; - BYTE* ip = (BYTE*)compressedBuffer; - BYTE* const iend = (BYTE*)compressedBuffer + cSize; - U64 crcDest; + { size_t const decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; + unsigned const maxBits = FUZ_highbit((U32)decodedBufferSize); + BYTE* const ostart = (BYTE*)decodedBuffer; + BYTE* op = ostart; + BYTE* const oend = ostart + COMPRESSIBLE_NOISE_LENGTH; + const BYTE* ip = (const BYTE*)compressedBuffer; + const BYTE* const iend = (const BYTE*)compressedBuffer + cSize; - LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) goto _output_error; + { LZ4F_errorCode_t const createError = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); + if (LZ4F_isError(createError)) goto _output_error; } DISPLAYLEVEL(3, "random segment sizes : \n"); while (ip < iend) { - unsigned nbBits = FUZ_rand(&randState) % maxBits; + unsigned const nbBits = FUZ_rand(&randState) % maxBits; size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; size_t oSize = oend-op; if (iSize > (size_t)(iend-ip)) iSize = iend-ip; - //DISPLAY("%7i : + %6i\n", (int)(ip-(BYTE*)compressedBuffer), (int)iSize); - errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; + { size_t const decompressError = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); + if (LZ4F_isError(decompressError)) goto _output_error; } op += oSize; ip += iSize; } - crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); - if (crcDest != crcOrig) goto _output_error; - DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize); + { size_t const decodedSize = (size_t)(op - ostart); + U64 const crcDest = XXH64(decodedBuffer, decodedSize, 1); + if (crcDest != crcOrig) goto _output_error; + DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize); + } - errorCode = LZ4F_freeDecompressionContext(dCtx); - if (LZ4F_isError(errorCode)) goto _output_error; + { LZ4F_errorCode_t const freeError = LZ4F_freeDecompressionContext(dCtx); + if (LZ4F_isError(freeError)) goto _output_error; } + dCtx = NULL; } DISPLAYLEVEL(3, "without checksum : \n"); @@ -385,15 +407,21 @@ int basicTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "Using 4 MB block : \n"); prefs.frameInfo.blockSizeID = LZ4F_max4MB; prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); + { size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs); + DISPLAYLEVEL(4, "dstCapacity = %u \n", (U32)dstCapacity) + cSize = LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs); + if (LZ4F_isError(cSize)) goto _output_error; + DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize); + } DISPLAYLEVEL(3, "without checksum : \n"); prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); + { size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs); + DISPLAYLEVEL(4, "dstCapacity = %u \n", (U32)dstCapacity) + cSize = LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs); + if (LZ4F_isError(cSize)) goto _output_error; + DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize); + } { size_t errorCode; BYTE* const ostart = (BYTE*)compressedBuffer; @@ -541,13 +569,13 @@ static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, un } -static const U32 srcDataLength = 9 MB; /* needs to be > 2x4MB to test large blocks */ - int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration_s) { unsigned testResult = 0; unsigned testNb = 0; + size_t const srcDataLength = 9 MB; /* needs to be > 2x4MB to test large blocks */ void* srcBuffer = NULL; + size_t const compressedBufferSize = LZ4F_compressFrameBound(srcDataLength, NULL); void* compressedBuffer = NULL; void* decodedBuffer = NULL; U32 coreRand = seed; @@ -556,7 +584,6 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi size_t result; clock_t const startClock = clock(); clock_t const clockDuration = duration_s * CLOCKS_PER_SEC; - XXH64_state_t xxh64; # define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } @@ -567,47 +594,39 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result); srcBuffer = malloc(srcDataLength); CHECK(srcBuffer==NULL, "srcBuffer Allocation failed"); - compressedBuffer = malloc(LZ4F_compressFrameBound(srcDataLength, NULL)); + compressedBuffer = malloc(compressedBufferSize); CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed"); decodedBuffer = calloc(1, srcDataLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */ CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed"); FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand); /* jump to requested testNb */ - for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); // sync randomizer + for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); /* sync randomizer */ /* main fuzzer test loop */ for ( ; (testNb < nbTests) || (clockDuration > FUZ_GetClockSpan(startClock)) ; testNb++) { U32 randState = coreRand ^ prime1; - unsigned BSId = 4 + (FUZ_rand(&randState) & 3); - unsigned BMId = FUZ_rand(&randState) & 1; - unsigned CCflag = FUZ_rand(&randState) & 1; - unsigned autoflush = (FUZ_rand(&randState) & 7) == 2; + unsigned const srcBits = (FUZ_rand(&randState) % (FUZ_highbit(srcDataLength-1) - 1)) + 1; + size_t const srcSize = (FUZ_rand(&randState) & ((1<<srcBits)-1)) + 1; + size_t const srcStartId = FUZ_rand(&randState) % (srcDataLength - srcSize); + const BYTE* const srcStart = (const BYTE*)srcBuffer + srcStartId; + unsigned const neverFlush = (FUZ_rand(&randState) & 15) == 1; + U64 const crcOrig = XXH64(srcStart, srcSize, 1); LZ4F_preferences_t prefs; - LZ4F_compressOptions_t cOptions; - LZ4F_decompressOptions_t dOptions; - unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(srcDataLength-1) - 1)) + 1; - size_t srcSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; - size_t srcStart = FUZ_rand(&randState) % (srcDataLength - srcSize); - U64 frameContentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0; + const LZ4F_preferences_t* prefsPtr = &prefs; size_t cSize; - U64 crcOrig; - LZ4F_preferences_t* prefsPtr = &prefs; (void)FUZ_rand(&coreRand); /* update seed */ memset(&prefs, 0, sizeof(prefs)); - memset(&cOptions, 0, sizeof(cOptions)); - memset(&dOptions, 0, sizeof(dOptions)); - prefs.frameInfo.blockMode = (LZ4F_blockMode_t)BMId; - prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)BSId; - prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)CCflag; - prefs.frameInfo.contentSize = frameContentSize; - prefs.autoFlush = autoflush; + prefs.frameInfo.blockMode = (LZ4F_blockMode_t)(FUZ_rand(&randState) & 1); + prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)(4 + (FUZ_rand(&randState) & 3)); + prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)(FUZ_rand(&randState) & 1); + prefs.frameInfo.contentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0; + prefs.autoFlush = neverFlush ? 0 : (FUZ_rand(&randState) & 7) == 2; prefs.compressionLevel = FUZ_rand(&randState) % 5; if ((FUZ_rand(&randState) & 0xF) == 1) prefsPtr = NULL; DISPLAYUPDATE(2, "\r%5u ", testNb); - crcOrig = XXH64((BYTE*)srcBuffer+srcStart, srcSize, 1); if ((FUZ_rand(&randState) & 0xFFF) == 0) { /* create a skippable frame (rare case) */ @@ -615,66 +634,72 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi FUZ_writeLE32(op, LZ4F_MAGIC_SKIPPABLE_START + (FUZ_rand(&randState) & 15)); FUZ_writeLE32(op+4, (U32)srcSize); cSize = srcSize+8; - } else if ((FUZ_rand(&randState) & 0xF) == 2) { - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(srcSize, prefsPtr), (char*)srcBuffer + srcStart, srcSize, prefsPtr); + } else if ((FUZ_rand(&randState) & 0xF) == 2) { /* single pass compression (simple) */ + cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(srcSize, prefsPtr), srcStart, srcSize, prefsPtr); CHECK(LZ4F_isError(cSize), "LZ4F_compressFrame failed : error %i (%s)", (int)cSize, LZ4F_getErrorName(cSize)); - } else { - const BYTE* ip = (const BYTE*)srcBuffer + srcStart; - const BYTE* const iend = ip + srcSize; + } else { /* multi-segments compression */ + const BYTE* ip = srcStart; + const BYTE* const iend = srcStart + srcSize; BYTE* op = (BYTE*)compressedBuffer; - BYTE* const oend = op + LZ4F_compressFrameBound(srcDataLength, NULL); - unsigned maxBits = FUZ_highbit((U32)srcSize); + BYTE* const oend = op + (neverFlush ? LZ4F_compressFrameBound(srcSize, prefsPtr) : compressedBufferSize); /* when flushes are possible, can't guarantee a max compressed size */ + unsigned const maxBits = FUZ_highbit((U32)srcSize); + LZ4F_compressOptions_t cOptions; + memset(&cOptions, 0, sizeof(cOptions)); result = LZ4F_compressBegin(cCtx, op, oend-op, prefsPtr); CHECK(LZ4F_isError(result), "Compression header failed (error %i)", (int)result); op += result; while (ip < iend) { - unsigned nbBitsSeg = FUZ_rand(&randState) % maxBits; - size_t iSize = (FUZ_rand(&randState) & ((1<<nbBitsSeg)-1)) + 1; - size_t oSize = LZ4F_compressBound(iSize, prefsPtr); - unsigned forceFlush = ((FUZ_rand(&randState) & 3) == 1); - if (iSize > (size_t)(iend-ip)) iSize = iend-ip; + unsigned const nbBitsSeg = FUZ_rand(&randState) % maxBits; + size_t const sampleMax = (FUZ_rand(&randState) & ((1<<nbBitsSeg)-1)) + 1; + size_t const iSize = MIN(sampleMax, (size_t)(iend-ip)); + size_t const oSize = LZ4F_compressBound(iSize, prefsPtr); cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1); result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions); - CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result); + CHECK(LZ4F_isError(result), "Compression failed (error %i : %s)", (int)result, LZ4F_getErrorName(result)); op += result; ip += iSize; - if (forceFlush) { - result = LZ4F_flush(cCtx, op, oend-op, &cOptions); - CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result); - op += result; - } + { unsigned const forceFlush = neverFlush ? 0 : ((FUZ_rand(&randState) & 3) == 1); + if (forceFlush) { + result = LZ4F_flush(cCtx, op, oend-op, &cOptions); + CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result); + op += result; + } } } + CHECK(op>=oend, "LZ4F_compressFrameBound overflow"); result = LZ4F_compressEnd(cCtx, op, oend-op, &cOptions); - CHECK(LZ4F_isError(result), "Compression completion failed (error %i)", (int)result); + CHECK(LZ4F_isError(result), "Compression completion failed (error %i : %s)", (int)result, LZ4F_getErrorName(result)); op += result; cSize = op-(BYTE*)compressedBuffer; + DISPLAYLEVEL(5, "\nCompressed %u bytes into %u \n", (U32)srcSize, (U32)cSize); } + /* multi-segments decompression */ { const BYTE* ip = (const BYTE*)compressedBuffer; const BYTE* const iend = ip + cSize; BYTE* op = (BYTE*)decodedBuffer; BYTE* const oend = op + srcDataLength; + unsigned const suggestedBits = FUZ_highbit((U32)cSize); + unsigned const maxBits = MAX(3, suggestedBits); + unsigned const nonContiguousDst = FUZ_rand(&randState) % 3; /* 0 : contiguous; 1 : non-contiguous; 2 : dst overwritten */ size_t totalOut = 0; - unsigned maxBits = FUZ_highbit((U32)cSize); - unsigned nonContiguousDst = (FUZ_rand(&randState) & 3) == 1; /* 0 : contiguous; 1 : non-contiguous; 2 : dst overwritten */ - nonContiguousDst += FUZ_rand(&randState) & nonContiguousDst; /* 0=>0; 1=>1,2 */ + XXH64_state_t xxh64; XXH64_reset(&xxh64, 1); - if (maxBits < 3) maxBits = 3; while (ip < iend) { unsigned const nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1; unsigned const nbBitsO = (FUZ_rand(&randState) % (maxBits)) + 1; - size_t iSize = (FUZ_rand(&randState) & ((1<<nbBitsI)-1)) + 1; - size_t oSize = (FUZ_rand(&randState) & ((1<<nbBitsO)-1)) + 2; - if (iSize > (size_t)(iend-ip)) iSize = iend-ip; - if (oSize > (size_t)(oend-op)) oSize = oend-op; + size_t const iSizeMax = (FUZ_rand(&randState) & ((1<<nbBitsI)-1)) + 1; + size_t iSize = MIN(iSizeMax, (size_t)(iend-ip)); + size_t const oSizeMax = (FUZ_rand(&randState) & ((1<<nbBitsO)-1)) + 2; + size_t oSize = MIN(oSizeMax, (size_t)(oend-op)); + LZ4F_decompressOptions_t dOptions; + memset(&dOptions, 0, sizeof(dOptions)); dOptions.stableDst = FUZ_rand(&randState) & 1; - if (nonContiguousDst==2) dOptions.stableDst = 0; + if (nonContiguousDst==2) dOptions.stableDst = 0; /* overwrite mode */ result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions); - if (result == (size_t)-LZ4F_ERROR_contentChecksum_invalid) - locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst); - CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s)", (int)result, LZ4F_getErrorName((LZ4F_errorCode_t)result)); + if (LZ4F_getErrorCode(result) == LZ4F_ERROR_contentChecksum_invalid) locateBuffDiff(srcStart, decodedBuffer, srcSize, nonContiguousDst); + CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s)", (int)result, LZ4F_getErrorName(result)); XXH64_update(&xxh64, op, (U32)oSize); totalOut += oSize; op += oSize; @@ -685,7 +710,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi CHECK(result != 0, "Frame decompression failed (error %i)", (int)result); if (totalOut) { /* otherwise, it's a skippable frame */ U64 const crcDecoded = XXH64_digest(&xxh64); - if (crcDecoded != crcOrig) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst); + if (crcDecoded != crcOrig) locateBuffDiff(srcStart, decodedBuffer, srcSize, nonContiguousDst); CHECK(crcDecoded != crcOrig, "Decompression corruption"); } } @@ -712,7 +737,7 @@ _output_error: } -int FUZ_usage(void) +int FUZ_usage(const char* programName) { DISPLAY( "Usage :\n"); DISPLAY( " %s [args]\n", programName); @@ -722,14 +747,14 @@ int FUZ_usage(void) DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n"); DISPLAY( " -s# : Select seed (default:prompt user)\n"); DISPLAY( " -t# : Select starting test number (default:0)\n"); - DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); + DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); DISPLAY( " -v : verbose\n"); DISPLAY( " -h : display help and exit\n"); return 0; } -int main(int argc, char** argv) +int main(int argc, const char** argv) { U32 seed=0; int seedset=0; @@ -739,15 +764,15 @@ int main(int argc, char** argv) int proba = FUZ_COMPRESSIBILITY_DEFAULT; int result=0; U32 duration=0; + const char* const programName = argv[0]; /* Check command line */ - programName = argv[0]; - for(argNb=1; argNb<argc; argNb++) { - char* argument = argv[argNb]; + for (argNb=1; argNb<argc; argNb++) { + const char* argument = argv[argNb]; if(!argument) continue; /* Protection if argument empty */ - /* Decode command (note : aggregated commands are allowed) */ + /* Decode command (note : aggregated short commands are allowed) */ if (argument[0]=='-') { if (!strcmp(argument, "--no-prompt")) { no_prompt=1; @@ -761,10 +786,10 @@ int main(int argc, char** argv) switch(*argument) { case 'h': - return FUZ_usage(); + return FUZ_usage(programName); case 'v': argument++; - displayLevel=4; + displayLevel++; break; case 'q': argument++; @@ -841,22 +866,22 @@ int main(int argc, char** argv) break; default: ; - return FUZ_usage(); + return FUZ_usage(programName); } } } } /* Get Seed */ - printf("Starting lz4frame tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION_STRING); + DISPLAY("Starting lz4frame tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION_STRING); if (!seedset) { time_t const t = time(NULL); U32 const h = XXH32(&t, sizeof(t), 1); seed = h % 10000; } - printf("Seed = %u\n", seed); - if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba); + DISPLAY("Seed = %u\n", seed); + if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba); if (nbTests<=0) nbTests=1; diff --git a/tests/fullbench.c b/tests/fullbench.c index fa37fa6..1cac1e0 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -229,9 +229,14 @@ static int local_LZ4_saveDict(const char* in, char* out, int inSize) return LZ4_saveDict(&LZ4_stream, out, inSize); } +static int local_LZ4_compress_default(const char* in, char* out, int inSize) +{ + return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize)); +} + static int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize) { - return LZ4_compress_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)-1); + return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize)-1); } static int local_LZ4_compress_default_large(const char* in, char* out, int inSize) @@ -276,22 +281,22 @@ static int local_LZ4_compress_fast_continue0(const char* in, char* out, int inSi static int local_LZ4_compress_withState(const char* in, char* out, int inSize) { - return LZ4_compress_withState(&LZ4_stream, in, out, inSize); + return LZ4_compress_fast_extState(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize), 1); } static int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize) { - return LZ4_compress_limitedOutput_withState(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize)-1); + return LZ4_compress_fast_extState(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize)-1, 1); } static int local_LZ4_compress_continue(const char* in, char* out, int inSize) { - return LZ4_compress_continue(&LZ4_stream, in, out, inSize); + return LZ4_compress_fast_continue(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize), 1); } static int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize) { - return LZ4_compress_limitedOutput_continue(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize)-1); + return LZ4_compress_fast_continue(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize)-1, 1); } #ifndef LZ4_DLL_IMPORT @@ -318,29 +323,34 @@ static int local_LZ4_saveDictHC(const char* in, char* out, int inSize) return LZ4_saveDictHC(&LZ4_streamHC, out, inSize); } +static int local_LZ4_compress_HC(const char* in, char* out, int inSize) +{ + return LZ4_compress_HC(in, out, inSize, LZ4_compressBound(inSize), 9); +} + static int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize) { - return LZ4_compressHC_withStateHC(&LZ4_streamHC, in, out, inSize); + return LZ4_compress_HC_extStateHC(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize), 9); } static int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize) { - return LZ4_compressHC_limitedOutput_withStateHC(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize)-1); + return LZ4_compress_HC_extStateHC(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize)-1, 9); } static int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize) { - return LZ4_compressHC_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)-1); + return LZ4_compress_HC(in, out, inSize, LZ4_compressBound(inSize)-1, 9); } static int local_LZ4_compressHC_continue(const char* in, char* out, int inSize) { - return LZ4_compressHC_continue(&LZ4_streamHC, in, out, inSize); + return LZ4_compress_HC_continue(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize)); } static int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize) { - return LZ4_compressHC_limitedOutput_continue(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize)-1); + return LZ4_compress_HC_continue(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize)-1); } @@ -519,7 +529,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) case 7 : compressionFunction = local_LZ4_compress_fast_extState0; compressorName = "LZ4_compress_fast_extState(0)"; break; case 8 : compressionFunction = local_LZ4_compress_fast_continue0; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_fast_continue(0)"; break; - case 10: compressionFunction = LZ4_compressHC; compressorName = "LZ4_compressHC"; break; + case 10: compressionFunction = local_LZ4_compress_HC; compressorName = "LZ4_compress_HC"; break; case 11: compressionFunction = local_LZ4_compressHC_limitedOutput; compressorName = "LZ4_compressHC_limitedOutput"; break; case 12 : compressionFunction = local_LZ4_compressHC_withStateHC; compressorName = "LZ4_compressHC_withStateHC"; break; case 13: compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; compressorName = "LZ4_compressHC_limitedOutput_withStateHC"; break; @@ -540,7 +550,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) LZ4_loadDictHC(&LZ4_streamHC, chunkP[0].origBuffer, chunkP[0].origSize); break; case 60: DISPLAY("Obsolete compression functions : \n"); continue; - case 61: compressionFunction = LZ4_compress; compressorName = "LZ4_compress"; break; + case 61: compressionFunction = local_LZ4_compress_default; compressorName = "LZ4_compress_default"; break; case 62: compressionFunction = local_LZ4_compress_limitedOutput; compressorName = "LZ4_compress_limitedOutput"; break; case 63: compressionFunction = local_LZ4_compress_withState; compressorName = "LZ4_compress_withState"; break; case 64: compressionFunction = local_LZ4_compress_limitedOutput_withState; compressorName = "LZ4_compress_limitedOutput_withState"; break; @@ -602,7 +612,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) } } for (chunkNb=0; chunkNb<nbChunks; chunkNb++) { - chunkP[chunkNb].compressedSize = LZ4_compress(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize); + chunkP[chunkNb].compressedSize = LZ4_compress_default(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize, maxCompressedChunkSize); if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress"), exit(1); } diff --git a/tests/fuzzer.c b/tests/fuzzer.c index bbf4e14..4d8d8dd 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -33,20 +33,14 @@ # pragma warning(disable : 4310) /* disable: C4310: constant char value > 127 */ #endif -/* S_ISREG & gettimeofday() are not supported by MSVC */ -#if defined(_MSC_VER) || defined(_WIN32) -# define FUZ_LEGACY_TIMER 1 -#endif - /*-************************************ -* Includes +* Dependencies **************************************/ #include <stdlib.h> #include <stdio.h> /* fgets, sscanf */ #include <string.h> /* strcmp */ #include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */ -#include "lz4.h" /* LZ4_VERSION_STRING */ #include "lz4hc.h" #define XXH_STATIC_LINKING_ONLY #include "xxhash.h" @@ -55,7 +49,7 @@ /*-************************************ * Basic Types **************************************/ -#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ +#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) # include <stdint.h> typedef uint8_t BYTE; typedef uint16_t U16; @@ -68,6 +62,7 @@ typedef unsigned short U16; typedef unsigned int U32; typedef signed int S32; typedef unsigned long long U64; +typedef size_t uintptr_t; /* true on most systems, except OpenVMS-64 (which doesn't need address overflow test) */ #endif @@ -91,11 +86,11 @@ typedef unsigned long long U64; /*-*************************************** * Macros *****************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAY(...) fprintf(stdout, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } static int g_displayLevel = 2; -static const clock_t g_refreshRate = CLOCKS_PER_SEC * 25 / 100; -static clock_t g_time = 0; + +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) /*-******************************************************* @@ -106,6 +101,17 @@ static clock_t FUZ_GetClockSpan(clock_t clockStart) return clock() - clockStart; /* works even if overflow; max span ~ 30mn */ } +static void FUZ_displayUpdate(unsigned testNb) +{ + static clock_t g_time = 0; + static const clock_t g_refreshRate = CLOCKS_PER_SEC / 5; + if ((FUZ_GetClockSpan(g_time) > g_refreshRate) || (g_displayLevel>=4)) { + g_time = clock(); + DISPLAY("\r%5u ", testNb); + fflush(stdout); + } +} + static U32 FUZ_rotl32(U32 u32, U32 nbBits) { return ((u32 << nbBits) | (u32 >> (32 - nbBits))); @@ -126,36 +132,29 @@ static U32 FUZ_rand(U32* src) #define FUZ_RANDLENGTH ( ((FUZ_rand(seed) >> 7) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15) static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, double proba, U32* seed) { - BYTE* BBuffer = (BYTE*)buffer; + BYTE* const BBuffer = (BYTE*)buffer; size_t pos = 0; - U32 P32 = (U32)(32768 * proba); + U32 const P32 = (U32)(32768 * proba); /* First Bytes */ while (pos < 20) BBuffer[pos++] = (BYTE)(FUZ_rand(seed)); - while (pos < bufferSize) - { + while (pos < bufferSize) { /* Select : Literal (noise) or copy (within 64K) */ - if (FUZ_RAND15BITS < P32) - { + if (FUZ_RAND15BITS < P32) { /* Copy (within 64K) */ - size_t match, d; - size_t length = FUZ_RANDLENGTH + 4; + size_t const length = FUZ_RANDLENGTH + 4; + size_t const d = MIN(pos+length, bufferSize); + size_t match; size_t offset = FUZ_RAND15BITS + 1; while (offset > pos) offset >>= 1; - d = pos + length; - while (d > bufferSize) d = bufferSize; match = pos - offset; while (pos < d) BBuffer[pos++] = BBuffer[match++]; - } - else - { + } else { /* Literal (noise) */ - size_t d; - size_t length = FUZ_RANDLENGTH; - d = pos + length; - if (d > bufferSize) d = bufferSize; + size_t const length = FUZ_RANDLENGTH; + size_t const d = MIN(pos+length, bufferSize); while (pos < d) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5); } } @@ -177,8 +176,7 @@ static int FUZ_AddressOverflow(void) DISPLAY("Overflow tests : "); /* Only possible in 32-bits */ - if (sizeof(void*)==8) - { + if (sizeof(void*)==8) { DISPLAY("64 bits mode : no overflow \n"); fflush(stdout); return 0; @@ -196,13 +194,13 @@ static int FUZ_AddressOverflow(void) buffers[nbBuff] = (char*)malloc(BLOCKSIZE_I134); if (buffers[nbBuff]==NULL) goto _endOfTests; - if (((size_t)buffers[nbBuff] > (size_t)0x80000000) && (!highAddress)) { + if (((uintptr_t)buffers[nbBuff] > (uintptr_t)0x80000000) && (!highAddress)) { DISPLAY("high address detected : "); fflush(stdout); highAddress=1; } - { size_t const sizeToGenerateOverflow = (size_t)(- ((size_t)buffers[nbBuff-1]) + 512); + { size_t const sizeToGenerateOverflow = (size_t)(- ((uintptr_t)buffers[nbBuff-1]) + 512); int const nbOf255 = (int)((sizeToGenerateOverflow / 255) + 1); char* const input = buffers[nbBuff-1]; char* output = buffers[nbBuff]; @@ -250,16 +248,6 @@ _overflowError: } -static void FUZ_displayUpdate(unsigned testNb) -{ - if ((FUZ_GetClockSpan(g_time) > g_refreshRate) | (g_displayLevel>=3)) { - g_time = clock(); - DISPLAY("\r%5u ", testNb); - if (g_displayLevel>=3) fflush(stdout); - } -} - - /*! FUZ_findDiff() : * find the first different byte between buff1 and buff2. * presumes buff1 != buff2. @@ -270,9 +258,9 @@ static void FUZ_findDiff(const void* buff1, const void* buff2) { const BYTE* const b1 = (const BYTE*)buff1; const BYTE* const b2 = (const BYTE*)buff2; - size_t i=0; - while (b1[i]==b2[i]) i++; - DISPLAY("Wrong Byte at position %u\n", (unsigned)i); + size_t u = 0; + while (b1[u]==b2[u]) u++; + DISPLAY("Wrong Byte at position %u \n", (unsigned)u); } @@ -282,86 +270,81 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c unsigned long long cbytes = 0; unsigned long long hcbytes = 0; unsigned long long ccbytes = 0; - void* CNBuffer; - char* compressedBuffer; - char* decodedBuffer; -# define FUZ_max LZ4_COMPRESSBOUND(LEN) - int ret; - unsigned cycleNb; -# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \ - printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; } -# define FUZ_DISPLAYTEST { testNb++; g_displayLevel<3 ? 0 : printf("%2u\b\b", testNb); if (g_displayLevel==4) fflush(stdout); } - void* stateLZ4 = malloc(LZ4_sizeofState()); - void* stateLZ4HC = malloc(LZ4_sizeofStateHC()); + void* const CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); + size_t const compressedBufferSize = LZ4_compressBound(FUZ_MAX_BLOCK_SIZE); + char* const compressedBuffer = (char*)malloc(compressedBufferSize); + char* const decodedBuffer = (char*)malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE); + void* const stateLZ4 = malloc(LZ4_sizeofState()); + void* const stateLZ4HC = malloc(LZ4_sizeofStateHC()); LZ4_stream_t LZ4dict; LZ4_streamHC_t LZ4dictHC; - U32 crcOrig, crcCheck; U32 coreRandState = seed; - U32 randState = coreRandState ^ PRIME3; - int result = 0; clock_t const clockStart = clock(); clock_t const clockDuration = (clock_t)duration_s * CLOCKS_PER_SEC; + int result = 0; + unsigned cycleNb; + +# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \ + printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; } +# define FUZ_DISPLAYTEST { testNb++; g_displayLevel>=4 ? printf("%2u\b\b", testNb), fflush(stdout) : 0; } /* init */ + if(!CNBuffer || !compressedBuffer || !decodedBuffer) { + DISPLAY("Not enough memory to start fuzzer tests"); + goto _output_error; + } memset(&LZ4dict, 0, sizeof(LZ4dict)); - - /* Create compressible test buffer */ - CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); - FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); - compressedBuffer = (char*)malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE)); - decodedBuffer = (char*)malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE); + { U32 randState = coreRandState ^ PRIME3; + FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); + } /* move to startCycle */ for (cycleNb = 0; cycleNb < startCycle; cycleNb++) { - (void)FUZ_rand(&coreRandState); + U32 randState = FUZ_rand(&coreRandState) ^ PRIME3; if (0) { /* some problems can be related to dictionary re-use; in this case, enable this loop */ - int dictSize, blockSize, blockStart; - char* dict; - char* block; + int const blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE; + int const blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); + int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; + int const dictSize = MIN(dictSizeRand, blockStart); + char* const block = ((char*)CNBuffer) + blockStart; + const char* const dict = block - dictSize; FUZ_displayUpdate(cycleNb); - randState = coreRandState ^ PRIME3; - blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE; - blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); - dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; - if (dictSize > blockStart) dictSize = blockStart; - block = ((char*)CNBuffer) + blockStart; - dict = block - dictSize; LZ4_loadDict(&LZ4dict, dict, dictSize); - LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); + LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, compressedBufferSize, 1); LZ4_loadDict(&LZ4dict, dict, dictSize); - LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); + LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, compressedBufferSize, 1); LZ4_loadDict(&LZ4dict, dict, dictSize); - LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); + LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, compressedBufferSize, 1); } } /* Main test loop */ - for (cycleNb = startCycle; (cycleNb < nbCycles) || (FUZ_GetClockSpan(clockStart) < clockDuration) ; cycleNb++) { + for (cycleNb = startCycle; + (cycleNb < nbCycles) || (FUZ_GetClockSpan(clockStart) < clockDuration); + cycleNb++) { U32 testNb = 0; - char* dict; - char* block; - int dictSize, blockSize, blockStart, compressedSize, HCcompressedSize; + U32 randState = FUZ_rand(&coreRandState) ^ PRIME3; + int const blockSize = (FUZ_rand(&randState) % (FUZ_MAX_BLOCK_SIZE-1)) + 1; + int const blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); + int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; + int const dictSize = MIN(dictSizeRand, blockStart); + char* const block = ((char*)CNBuffer) + blockStart; + const char* dict = block - dictSize; + int compressedSize, HCcompressedSize; int blockContinueCompressedSize; + U32 const crcOrig = XXH32(block, blockSize, 0); + U32 crcCheck; + int ret; FUZ_displayUpdate(cycleNb); - (void)FUZ_rand(&coreRandState); - randState = coreRandState ^ PRIME3; - - /* Select block to test */ - blockSize = (FUZ_rand(&randState) % (FUZ_MAX_BLOCK_SIZE-1)) + 1; - blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); - dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; - if (dictSize > blockStart) dictSize = blockStart; - block = ((char*)CNBuffer) + blockStart; - dict = block - dictSize; /* Compression tests */ /* Test compression destSize */ FUZ_DISPLAYTEST; { int srcSize = blockSize; - int targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7; + int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7; char endCheck = FUZ_rand(&randState) & 255; compressedBuffer[targetSize] = endCheck; ret = LZ4_compress_destSize(block, compressedBuffer, &srcSize, targetSize); @@ -370,20 +353,19 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_destSize() fed more than src buffer !"); DISPLAYLEVEL(5, "destSize : %7i/%7i; content%7i/%7i ", ret, targetSize, srcSize, blockSize); if (targetSize>0) { - FUZ_CHECKTEST((ret==0), "LZ4_compress_destSize() compression failed"); /* check correctness */ + U32 const crcBase = XXH32(block, srcSize, 0); + char const canary = FUZ_rand(&randState) & 255; + FUZ_CHECKTEST((ret==0), "LZ4_compress_destSize() compression failed"); FUZ_DISPLAYTEST; - - crcOrig = XXH32(block, srcSize, 0); compressedSize = ret; - endCheck = FUZ_rand(&randState) & 255; - decodedBuffer[srcSize] = endCheck; + decodedBuffer[srcSize] = canary; ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize); FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compress_destSize"); FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data"); - FUZ_CHECKTEST(decodedBuffer[srcSize] != endCheck, "LZ4_decompress_safe() overwrite dst buffer !"); - crcCheck = XXH32(decodedBuffer, srcSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe() corrupted decoded data"); + FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe() overwrite dst buffer !"); + { U32 const crcDec = XXH32(decodedBuffer, srcSize, 0); + FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); } DISPLAYLEVEL(5, " OK \n"); } @@ -393,30 +375,28 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Test compression HC */ FUZ_DISPLAYTEST; - ret = LZ4_compressHC(block, compressedBuffer, blockSize); + ret = LZ4_compress_HC(block, compressedBuffer, blockSize, compressedBufferSize, 9); FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed"); HCcompressedSize = ret; /* Test compression HC using external state */ FUZ_DISPLAYTEST; - ret = LZ4_compressHC_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize); + ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, compressedBufferSize, 9); FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed"); /* Test compression using external state */ FUZ_DISPLAYTEST; - ret = LZ4_compress_withState(stateLZ4, block, compressedBuffer, blockSize); + ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, compressedBufferSize, 9); FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed"); /* Test compression */ FUZ_DISPLAYTEST; - ret = LZ4_compress(block, compressedBuffer, blockSize); + ret = LZ4_compress_default(block, compressedBuffer, blockSize, compressedBufferSize); FUZ_CHECKTEST(ret==0, "LZ4_compress() failed"); compressedSize = ret; /* Decompression tests */ - crcOrig = XXH32(block, blockSize, 0); - /* Test decoding with output size being exactly what's necessary => must work */ FUZ_DISPLAYTEST; ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize); @@ -502,22 +482,22 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Test compression with output size being exactly what's necessary (should work) */ FUZ_DISPLAYTEST; - ret = LZ4_compress_limitedOutput(block, compressedBuffer, blockSize, compressedSize); + ret = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize); FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space"); /* Test compression with output size being exactly what's necessary and external state (should work) */ FUZ_DISPLAYTEST; - ret = LZ4_compress_limitedOutput_withState(stateLZ4, block, compressedBuffer, blockSize, compressedSize); + ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, compressedSize, 1); FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput_withState() failed despite sufficient space"); /* Test HC compression with output size being exactly what's necessary (should work) */ FUZ_DISPLAYTEST; - ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize); + ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, 9); FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space"); /* Test HC compression with output size being exactly what's necessary (should work) */ FUZ_DISPLAYTEST; - ret = LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize); + ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, 9); FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space"); /* Test compression with missing bytes into output buffer => must fail */ @@ -526,7 +506,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c if (missingBytes >= compressedSize) missingBytes = compressedSize-1; missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ compressedBuffer[compressedSize-missingBytes] = 0; - ret = LZ4_compress_limitedOutput(block, compressedBuffer, blockSize, compressedSize-missingBytes); + ret = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize-missingBytes); FUZ_CHECKTEST(ret, "LZ4_compress_limitedOutput should have failed (output buffer too small by %i byte)", missingBytes); FUZ_CHECKTEST(compressedBuffer[compressedSize-missingBytes], "LZ4_compress_limitedOutput overran output buffer ! (%i missingBytes)", missingBytes) } @@ -537,7 +517,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c if (missingBytes >= HCcompressedSize) missingBytes = HCcompressedSize-1; missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ compressedBuffer[HCcompressedSize-missingBytes] = 0; - ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes); + ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, 9); FUZ_CHECKTEST(ret, "LZ4_compressHC_limitedOutput should have failed (output buffer too small by %i byte)", missingBytes); FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-missingBytes], "LZ4_compressHC_limitedOutput overran output buffer ! (%i missingBytes)", missingBytes) } @@ -551,8 +531,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_DISPLAYTEST; { LZ4_stream_t LZ4_stream; LZ4_resetStream(&LZ4_stream); - LZ4_compress_continue (&LZ4_stream, dict, compressedBuffer, dictSize); /* Just to fill hash tables */ - blockContinueCompressedSize = LZ4_compress_continue (&LZ4_stream, block, compressedBuffer, blockSize); + LZ4_compress_fast_continue (&LZ4_stream, dict, compressedBuffer, dictSize, compressedBufferSize, 1); /* Just to fill hash tables */ + blockContinueCompressedSize = LZ4_compress_fast_continue (&LZ4_stream, block, compressedBuffer, blockSize, compressedBufferSize, 1); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed"); } @@ -581,17 +561,17 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c dict -= (FUZ_rand(&randState) & 0xF) + 1; /* Separation, so it is an ExtDict */ if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; LZ4_loadDict(&LZ4dict, dict, dictSize); - blockContinueCompressedSize = LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); + blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, compressedBufferSize, 1); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed"); FUZ_DISPLAYTEST; LZ4_loadDict(&LZ4dict, dict, dictSize); - ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); + ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1); FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize); FUZ_DISPLAYTEST; LZ4_loadDict(&LZ4dict, dict, dictSize); - ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize); + ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer"); @@ -600,18 +580,17 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c decodedBuffer[blockSize] = 0; ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input"); - FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size") - crcCheck = XXH32(decodedBuffer, blockSize, 0); - if (crcCheck!=crcOrig) - FUZ_findDiff(block, decodedBuffer); + FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size"); + crcCheck = XXH32(decodedBuffer, blockSize, 0); + if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); FUZ_DISPLAYTEST; decodedBuffer[blockSize] = 0; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); - FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size") - crcCheck = XXH32(decodedBuffer, blockSize, 0); + FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); + crcCheck = XXH32(decodedBuffer, blockSize, 0); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); FUZ_DISPLAYTEST; @@ -641,17 +620,17 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; LZ4_resetStreamHC (&LZ4dictHC, FUZ_rand(&randState) & 0x7); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); - blockContinueCompressedSize = LZ4_compressHC_continue(&LZ4dictHC, block, compressedBuffer, blockSize); + blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, compressedBufferSize); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compressHC_continue failed"); FUZ_DISPLAYTEST; LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); - ret = LZ4_compressHC_limitedOutput_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); + ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); FUZ_CHECKTEST(ret>0, "LZ4_compressHC_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer"); FUZ_DISPLAYTEST; LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); - ret = LZ4_compressHC_limitedOutput_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize); + ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer"); @@ -723,7 +702,6 @@ static void FUZ_unitTests(void) { LZ4_stream_t* statePtr; LZ4_stream_t streamingState; U64 crcOrig; - U64 crcNew; int result; /* Allocation test */ @@ -734,13 +712,13 @@ static void FUZ_unitTests(void) /* simple compression test */ crcOrig = XXH64(testInput, testCompressedSize, 0); LZ4_resetStream(&streamingState); - result = LZ4_compress_limitedOutput_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1); + result = LZ4_compress_fast_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1, 1); FUZ_CHECKTEST(result==0, "LZ4_compress_limitedOutput_continue() compression failed"); result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize); FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); - crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); + { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); } /* ring buffer test */ { XXH64_state_t xxhOrig; @@ -764,15 +742,15 @@ static void FUZ_unitTests(void) crcOrig = XXH64_digest(&xxhOrig); memcpy (ringBuffer + rNext, testInput + iNext, messageSize); - result = LZ4_compress_limitedOutput_continue(&streamingState, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); + result = LZ4_compress_fast_continue(&streamingState, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize, 1); FUZ_CHECKTEST(result==0, "LZ4_compress_limitedOutput_continue() compression failed"); result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize); FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed"); XXH64_update(&xxhNew, testVerify + dNext, messageSize); - crcNew = XXH64_digest(&xxhNew); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); + { U64 const crcNew = XXH64_digest(&xxhNew); + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); } /* prepare next message */ iNext += messageSize; @@ -789,7 +767,6 @@ static void FUZ_unitTests(void) { LZ4_streamHC_t* sp; LZ4_streamHC_t sHC; U64 crcOrig; - U64 crcNew; int result; /* Allocation test */ @@ -800,25 +777,25 @@ static void FUZ_unitTests(void) /* simple HC compression test */ crcOrig = XXH64(testInput, testCompressedSize, 0); LZ4_resetStreamHC(&sHC, 0); - result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1); + result = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize); FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); - crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); + { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); } /* simple dictionary HC compression test */ crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); LZ4_resetStreamHC(&sHC, 0); LZ4_loadDictHC(&sHC, testInput, 64 KB); - result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); + result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 64 KB); FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed"); - crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() simple dictionary decompression test : corruption"); + { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() simple dictionary decompression test : corruption"); } /* multiple HC compression test with dictionary */ { int result1, result2; @@ -826,30 +803,30 @@ static void FUZ_unitTests(void) crcOrig = XXH64(testInput + segSize, testCompressedSize, 0); LZ4_resetStreamHC(&sHC, 0); LZ4_loadDictHC(&sHC, testInput, segSize); - result1 = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1); + result1 = LZ4_compress_HC_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1); FUZ_CHECKTEST(result1==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result1); - result2 = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + 2*segSize, testCompressed+result1, segSize, segSize-1); + result2 = LZ4_compress_HC_continue(&sHC, testInput + 2*segSize, testCompressed+result1, segSize, segSize-1); FUZ_CHECKTEST(result2==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result2); result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result1, segSize, testInput, segSize); FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 1 failed"); result = LZ4_decompress_safe_usingDict(testCompressed+result1, testVerify+segSize, result2, segSize, testInput, 2*segSize); FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 2 failed"); - crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); + { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); } } /* remote dictionary HC compression test */ crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); LZ4_resetStreamHC(&sHC, 0); LZ4_loadDictHC(&sHC, testInput, 32 KB); - result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); + result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result); result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 32 KB); FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe_usingDict() decompression failed following remote dictionary HC compression test"); - crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() decompression corruption"); + { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() decompression corruption"); } /* multiple HC compression with ext. dictionary */ { XXH64_state_t crcOrigState; @@ -871,19 +848,16 @@ static void FUZ_unitTests(void) while (segStart + segSize < testInputSize) { XXH64_update(&crcOrigState, testInput + segStart, segSize); crcOrig = XXH64_digest(&crcOrigState); - result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + segStart, testCompressed, segSize, LZ4_compressBound(segSize)); + result = LZ4_compress_HC_continue(&sHC, testInput + segStart, testCompressed, segSize, LZ4_compressBound(segSize)); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); result = LZ4_decompress_safe_usingDict(testCompressed, dst, result, segSize, dict, dictSize); FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %i failed", segNb); XXH64_update(&crcNewState, dst, segSize); - crcNew = XXH64_digest(&crcNewState); - if (crcOrig!=crcNew) { - size_t c=0; - while (dst[c] == testInput[segStart+c]) c++; - DISPLAY("Bad decompression at %u / %u \n", (U32)c, (U32)segSize); + { U64 const crcNew = XXH64_digest(&crcNewState); + if (crcOrig != crcNew) FUZ_findDiff(dst, testInput+segStart); + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() part %i corruption", segNb); } - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() part %i corruption", segNb); dict = dst; //dict = testInput + segStart; @@ -919,15 +893,15 @@ static void FUZ_unitTests(void) crcOrig = XXH64_digest(&xxhOrig); memcpy (ringBuffer + rNext, testInput + iNext, messageSize); - result = LZ4_compressHC_limitedOutput_continue(&sHC, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); + result = LZ4_compress_HC_continue(&sHC, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize); FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed"); XXH64_update(&xxhNew, testVerify + dNext, messageSize); - crcNew = XXH64_digest(&xxhNew); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); + { U64 const crcNew = XXH64_digest(&xxhNew); + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); } /* prepare next message */ iNext += messageSize; @@ -965,15 +939,15 @@ static void FUZ_unitTests(void) XXH64_update(&xxhOrig, testInput + iNext, messageSize); crcOrig = XXH64_digest(&xxhOrig); - result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); + result = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize); FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed"); XXH64_update(&xxhNew, testVerify + dNext, messageSize); - crcNew = XXH64_digest(&xxhNew); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); + { U64 const crcNew = XXH64_digest(&xxhNew); + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); } /* prepare next message */ dNext += messageSize; @@ -987,18 +961,17 @@ static void FUZ_unitTests(void) XXH64_update(&xxhOrig, testInput + iNext, messageSize); crcOrig = XXH64_digest(&xxhOrig); - result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); + result = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize); FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed"); XXH64_update(&xxhNew, testVerify + dNext, messageSize); - crcNew = XXH64_digest(&xxhNew); - if (crcOrig != crcNew) - FUZ_findDiff(testInput + iNext, testVerify + dNext); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption during small decoder-side ring buffer test"); - + { U64 const crcNew = XXH64_digest(&xxhNew); + if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, testVerify + dNext); + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption during small decoder-side ring buffer test"); + } /* prepare next message */ dNext += messageSize; totalMessageSize += messageSize; @@ -1016,7 +989,7 @@ _output_error: } -static int FUZ_usage(char* programName) +static int FUZ_usage(const char* programName) { DISPLAY( "Usage :\n"); DISPLAY( " %s [args]\n", programName); @@ -1034,21 +1007,21 @@ static int FUZ_usage(char* programName) } -int main(int argc, char** argv) +int main(int argc, const char** argv) { - U32 seed=0; - int seedset=0; + U32 seed = 0; + int seedset = 0; int argNb; int nbTests = NB_ATTEMPTS; int testNb = 0; int proba = FUZ_COMPRESSIBILITY_DEFAULT; int pause = 0; - char* programName = argv[0]; + const char* programName = argv[0]; U32 duration = 0; /* Check command line */ for(argNb=1; argNb<argc; argNb++) { - char* argument = argv[argNb]; + const char* argument = argv[argNb]; if(!argument) continue; // Protection if argument empty @@ -1065,7 +1038,7 @@ int main(int argc, char** argv) case 'v': /* verbose mode */ argument++; - g_displayLevel=4; + g_displayLevel++; break; case 'p': /* pause at the end */ |