diff options
| -rw-r--r-- | doc/lz4_Frame_format.md | 3 | ||||
| -rw-r--r-- | doc/lz4_manual.html | 34 | ||||
| -rw-r--r-- | doc/lz4frame_manual.html | 114 | ||||
| -rw-r--r-- | examples/frameCompress.c | 537 | ||||
| -rw-r--r-- | lib/lz4.c | 7 | ||||
| -rw-r--r-- | lib/lz4.h | 38 | ||||
| -rw-r--r-- | lib/lz4frame.c | 434 | ||||
| -rw-r--r-- | lib/lz4frame.h | 115 | ||||
| -rw-r--r-- | lib/lz4frame_static.h | 54 | ||||
| -rw-r--r-- | lib/lz4hc.c | 24 | ||||
| -rw-r--r-- | lib/lz4hc.h | 23 | ||||
| -rw-r--r-- | tests/frametest.c | 302 |
12 files changed, 1020 insertions, 665 deletions
diff --git a/doc/lz4_Frame_format.md b/doc/lz4_Frame_format.md index bae2b06..77454b2 100644 --- a/doc/lz4_Frame_format.md +++ b/doc/lz4_Frame_format.md @@ -237,8 +237,7 @@ __Header Checksum__ One-byte checksum of combined descriptor fields, including optional ones. The value is the second byte of `xxh32()` : ` (xxh32()>>8) & 0xFF ` -using zero as a seed, -and the full Frame Descriptor as an input +using zero as a seed, and the full Frame Descriptor as an input (including optional fields when they are present). A wrong checksum indicates an error in the descriptor. Header checksum is informational and can be skipped. diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html index fecd8cd..9ab1984 100644 --- a/doc/lz4_manual.html +++ b/doc/lz4_manual.html @@ -175,8 +175,8 @@ int LZ4_freeStream (LZ4_stream_t* streamPtr); Important : Previous data blocks are assumed to remain present and unmodified ! 'dst' buffer must be already allocated. If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. - If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero. - After an error, the stream status is invalid, and it can only be reset or freed. + If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function @return==0. + After an error, the stream status is invalid, it can only be reset or freed. </p></pre><BR> @@ -205,20 +205,20 @@ int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); <pre><b>int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); -</b><p> These decoding functions allow decompression of multiple blocks in "streaming" mode. - Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) - In the case of a ring buffers, decoding buffer must be either : - - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) - In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). - - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. - maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. - In which case, encoding and decoding buffers do not need to be synchronized, - and encoding ring buffer can have any size, including small ones ( < 64 KB). - - _At least_ 64 KB + 8 bytes + maxBlockSize. - In which case, encoding and decoding buffers do not need to be synchronized, - and encoding ring buffer can have any size, including larger than decoding buffer. - Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, - and indicate where it is saved using LZ4_setStreamDecode() +</b><p> These decoding functions allow decompression of multiple blocks in "streaming" mode. + Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) + In the case of a ring buffers, decoding buffer must be either : + - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) + In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). + - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. + maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. + In which case, encoding and decoding buffers do not need to be synchronized, + and encoding ring buffer can have any size, including small ones ( < 64 KB). + - _At least_ 64 KB + 8 bytes + maxBlockSize. + In which case, encoding and decoding buffers do not need to be synchronized, + and encoding ring buffer can have any size, including larger than decoding buffer. + Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, + and indicate where it is saved using LZ4_setStreamDecode() </p></pre><BR> <pre><b>int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); @@ -277,7 +277,7 @@ union LZ4_stream_u { init this structure before first use. note : only use in association with static linking ! this definition is not API/ABI safe, - and may change in a future version ! + it may change in a future version ! </p></pre><BR> diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html index 87750a1..2a625a6 100644 --- a/doc/lz4frame_manual.html +++ b/doc/lz4frame_manual.html @@ -69,12 +69,13 @@ } LZ4F_frameType_t; </b></pre><BR> <pre><b>typedef struct { - LZ4F_blockSizeID_t blockSizeID; </b>/* max64KB, max256KB, max1MB, max4MB ; 0 == default */<b> - LZ4F_blockMode_t blockMode; </b>/* blockLinked, blockIndependent ; 0 == default */<b> - LZ4F_contentChecksum_t contentChecksumFlag; </b>/* noContentChecksum, contentChecksumEnabled ; 0 == default */<b> - LZ4F_frameType_t frameType; </b>/* LZ4F_frame, skippableFrame ; 0 == default */<b> - unsigned long long contentSize; </b>/* Size of uncompressed (original) content ; 0 == unknown */<b> - unsigned reserved[2]; </b>/* must be zero for forward compatibility */<b> + LZ4F_blockSizeID_t blockSizeID; </b>/* max64KB, max256KB, max1MB, max4MB ; 0 == default */<b> + LZ4F_blockMode_t blockMode; </b>/* blockLinked, blockIndependent ; 0 == default */<b> + LZ4F_contentChecksum_t contentChecksumFlag; </b>/* noContentChecksum, contentChecksumEnabled ; 0 == default */<b> + LZ4F_frameType_t frameType; </b>/* LZ4F_frame, skippableFrame ; 0 == default */<b> + unsigned long long contentSize; </b>/* Size of uncompressed content ; 0 == unknown */<b> + unsigned dictID; </b>/* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */<b> + unsigned reserved[1]; </b>/* must be zero for forward compatibility */<b> } LZ4F_frameInfo_t; </b><p> makes it possible to supply detailed frame parameters to the stream interface. It's not required to set all fields, as long as the structure was initially memset() to zero. @@ -84,7 +85,7 @@ <pre><b>typedef struct { LZ4F_frameInfo_t frameInfo; int compressionLevel; </b>/* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */<b> - unsigned autoFlush; </b>/* 1 == always flush (reduce usage of tmp buffer) */<b> + unsigned autoFlush; </b>/* 1 == always flush, to reduce usage of internal buffers */<b> unsigned reserved[4]; </b>/* must be zero for forward compatibility */<b> } LZ4F_preferences_t; </b><p> makes it possible to supply detailed compression parameters to the stream interface. @@ -95,17 +96,17 @@ <a name="Chapter5"></a><h2>Simple compression function</h2><pre></pre> <pre><b>size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr); -</b><p> Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences. - Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression. +</b><p> Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences. + Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression. </p></pre><BR> -<pre><b>size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr); -</b><p> Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.1 - An important rule is that dstBuffer MUST be large enough (dstCapacity) to store the result in worst case situation. - This value is supplied by LZ4F_compressFrameBound(). - If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode). - The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. +<pre><b>size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_preferences_t* preferencesPtr); +</b><p> Compress an entire srcBuffer into a valid LZ4 frame. + dstCapacity MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). + The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. @return : number of bytes written into dstBuffer. or an error code if it fails (can be tested using LZ4F_isError()) @@ -133,9 +134,11 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); <a name="Chapter8"></a><h2>Compression</h2><pre></pre> -<pre><b>size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* prefsPtr); -</b><p> will write the frame header into dstBuffer. - dstCapacity must be large enough to store the header. Maximum header size is LZ4F_HEADER_SIZE_MAX bytes. +<pre><b>size_t LZ4F_compressBegin(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_preferences_t* prefsPtr); +</b><p> will write the frame header into dstBuffer. + dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. `prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default. @return : number of bytes written into dstBuffer for the header or an error code (which can be tested using LZ4F_isError()) @@ -173,28 +176,28 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); </p></pre><BR> <pre><b>size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr); -</b><p> To properly finish an LZ4 frame, invoke LZ4F_compressEnd(). - It will flush whatever data remained within `cctx` (like LZ4_flush()) - and properly finalize the frame, with an endMark and a checksum. +</b><p> To properly finish an LZ4 frame, invoke LZ4F_compressEnd(). + It will flush whatever data remained within `cctx` (like LZ4_flush()) + and properly finalize the frame, with an endMark and a checksum. `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default. @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled) or an error code if it fails (which can be tested using LZ4F_isError()) - A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task. + A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task. </p></pre><BR> <a name="Chapter9"></a><h2>Decompression functions</h2><pre></pre> <pre><b>typedef struct { - unsigned stableDst; </b>/* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */<b> - unsigned reserved[3]; + unsigned stableDst; </b>/* pledge that at least 64KB+64Bytes of previously decompressed data remain unmodifed where it was decoded. This optimization skips storage operations in tmp buffers */<b> + unsigned reserved[3]; </b>/* must be set to zero for forward compatibility */<b> } LZ4F_decompressOptions_t; </b></pre><BR> <pre><b>LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version); LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx); -</b><p> Create an LZ4F_decompressionContext_t object, which will be used to track all decompression operations. - The version provided MUST be LZ4F_VERSION. It is intended to track potential breaking differences between different versions. - The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext_t object. +</b><p> Create an LZ4F_dctx object, to track all decompression operations. + The version provided MUST be LZ4F_VERSION. + The function provides a pointer to an allocated and initialized LZ4F_dctx object. The result is an errorCode, which can be tested using LZ4F_isError(). dctx memory can be released using LZ4F_freeDecompressionContext(); The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released. @@ -207,22 +210,25 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx); <pre><b>size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoPtr, const void* srcBuffer, size_t* srcSizePtr); -</b><p> This function extracts frame parameters (such as max blockSize, frame checksum, etc.). - Its usage is optional. Extracted information can be useful for allocation purposes, typically. - This function works in 2 situations : - - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process. +</b><p> This function extracts frame parameters (max blockSize, dictID, etc.). + Its usage is optional. + Extracted information is typically useful for allocation and dictionary. + This function works in 2 situations : + - At the beginning of a new frame, in which case + it will decode information from `srcBuffer`, starting the decoding process. Input size must be large enough to successfully decode the entire frame header. Frame header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes. It's allowed to provide more input data than this minimum. - After decoding has been started. In which case, no input is read, frame parameters are extracted from dctx. - If decoding has just started, but not yet extracted information from header, LZ4F_getFrameInfo() will fail. - The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). - Decompression must resume from (srcBuffer + *srcSizePtr). + - If decoding has barely started, but not yet extracted information from header, + LZ4F_getFrameInfo() will fail. + The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). + Decompression must resume from (srcBuffer + *srcSizePtr). @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call, - or an error code which can be tested using LZ4F_isError() - note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped. - note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure. + or an error code which can be tested using LZ4F_isError(). + note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely. + note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure. </p></pre><BR> @@ -230,28 +236,28 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx); void* dstBuffer, size_t* dstSizePtr, const void* srcBuffer, size_t* srcSizePtr, const LZ4F_decompressOptions_t* dOptPtr); -</b><p> Call this function repetitively to regenerate data compressed within `srcBuffer`. - The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr. +</b><p> Call this function repetitively to regenerate compressed data from `srcBuffer`. + The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr. - The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). + The number of bytes regenerated into dstBuffer is provided within *dstSizePtr (necessarily <= original value). - The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). - Number of bytes read can be < number of bytes provided, meaning there is some more data to decode. - It typically happens when dstBuffer is not large enough to contain all decoded data. - Remaining data will have to be presented again in a subsequent invocation. + The number of bytes consumed from srcBuffer is provided within *srcSizePtr (necessarily <= original value). + Number of bytes consumed can be < number of bytes provided. + It typically happens when dstBuffer is not large enough to contain all decoded data. + Unconsumed source data must be presented again in subsequent invocations. `dstBuffer` content is expected to be flushed between each invocation, as its content will be overwritten. - `dstBuffer` can be changed at will between each consecutive function invocation. + `dstBuffer` itself can be changed at will between each consecutive function invocation. - @return is an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call. - Schematically, it's the size of the current (or remaining) compressed block + header of next block. - Respecting the hint provides some small speed benefit, because it skips intermediate buffers. - This is just a hint though, it's always possible to provide any srcSize. - When a frame is fully decoded, @return will be 0 (no more data expected). - If decompression failed, @return is an error code, which can be tested using LZ4F_isError(). + @return : an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call. + Schematically, it's the size of the current (or remaining) compressed block + header of next block. + Respecting the hint provides some small speed benefit, because it skips intermediate buffers. + This is just a hint though, it's always possible to provide any srcSize. + When a frame is fully decoded, @return will be 0 (no more data expected). + If decompression failed, @return is an error code, which can be tested using LZ4F_isError(). - After a frame is fully decoded, dctx can be used again to decompress another frame. - After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state. + After a frame is fully decoded, dctx can be used again to decompress another frame. + After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state. </p></pre><BR> @@ -259,7 +265,7 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx); </b><p> In case of an error, the context is left in "undefined" state. In which case, it's necessary to reset it, before re-using it. This method can also be used to abruptly stop an unfinished decompression, - and start a new with the same context. + and start a new one using the same context. </p></pre><BR> </html> diff --git a/examples/frameCompress.c b/examples/frameCompress.c index 75f1576..0a42fe3 100644 --- a/examples/frameCompress.c +++ b/examples/frameCompress.c @@ -13,299 +13,300 @@ #define LZ4_FOOTER_SIZE 4 static const LZ4F_preferences_t lz4_preferences = { - { LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame, 0, { 0, 0 } }, - 0, /* compression level */ - 0, /* autoflush */ - { 0, 0, 0, 0 }, /* reserved, must be set to 0 */ + { LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame, + 0 /* content size unknown */, 0 /* no dictID */ , { 0 } /* reserved */ }, + 0, /* compression level */ + 0, /* autoflush */ + { 0, 0, 0, 0 }, /* reserved, must be set to 0 */ }; static size_t compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) { - LZ4F_errorCode_t r; - LZ4F_compressionContext_t ctx; - char *src, *buf = NULL; - size_t size, n, k, count_in = 0, count_out, offset = 0, frame_size; - - r = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); - if (LZ4F_isError(r)) { - printf("Failed to create context: error %zu\n", r); - return 1; - } - r = 1; - - src = malloc(BUF_SIZE); - if (!src) { - printf("Not enough memory\n"); - goto cleanup; - } - - frame_size = LZ4F_compressBound(BUF_SIZE, &lz4_preferences); - size = frame_size + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE; - buf = malloc(size); - if (!buf) { - printf("Not enough memory\n"); - goto cleanup; - } - - n = offset = count_out = LZ4F_compressBegin(ctx, buf, size, &lz4_preferences); - if (LZ4F_isError(n)) { - printf("Failed to start compression: error %zu\n", n); - goto cleanup; - } - - printf("Buffer size is %zu bytes, header size %zu bytes\n", size, n); - - for (;;) { - k = fread(src, 1, BUF_SIZE, in); - if (k == 0) - break; - count_in += k; - - n = LZ4F_compressUpdate(ctx, buf + offset, size - offset, src, k, NULL); - if (LZ4F_isError(n)) { - printf("Compression failed: error %zu\n", n); - goto cleanup; - } - - offset += n; - count_out += n; - if (size - offset < frame_size + LZ4_FOOTER_SIZE) { - printf("Writing %zu bytes\n", offset); - - k = fwrite(buf, 1, offset, out); - if (k < offset) { - if (ferror(out)) - printf("Write failed\n"); - else - printf("Short write\n"); - goto cleanup; - } - - offset = 0; - } - } - - n = LZ4F_compressEnd(ctx, buf + offset, size - offset, NULL); - if (LZ4F_isError(n)) { - printf("Failed to end compression: error %zu\n", n); - goto cleanup; - } - - offset += n; - count_out += n; - printf("Writing %zu bytes\n", offset); - - k = fwrite(buf, 1, offset, out); - if (k < offset) { - if (ferror(out)) - printf("Write failed\n"); - else - printf("Short write\n"); - goto cleanup; - } - - *size_in = count_in; - *size_out = count_out; - r = 0; + LZ4F_errorCode_t r; + LZ4F_compressionContext_t ctx; + char *src, *buf = NULL; + size_t size, n, k, count_in = 0, count_out, offset = 0, frame_size; + + r = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); + if (LZ4F_isError(r)) { + printf("Failed to create context: error %zu\n", r); + return 1; + } + r = 1; /* function result; 1 == error, by default (early exit) */ + + src = malloc(BUF_SIZE); + if (!src) { + printf("Not enough memory\n"); + goto cleanup; + } + + frame_size = LZ4F_compressBound(BUF_SIZE, &lz4_preferences); + size = frame_size + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE; + buf = malloc(size); + if (!buf) { + printf("Not enough memory\n"); + goto cleanup; + } + + n = offset = count_out = LZ4F_compressBegin(ctx, buf, size, &lz4_preferences); + if (LZ4F_isError(n)) { + printf("Failed to start compression: error %zu\n", n); + goto cleanup; + } + + printf("Buffer size is %zu bytes, header size %zu bytes\n", size, n); + + for (;;) { + k = fread(src, 1, BUF_SIZE, in); + if (k == 0) + break; + count_in += k; + + n = LZ4F_compressUpdate(ctx, buf + offset, size - offset, src, k, NULL); + if (LZ4F_isError(n)) { + printf("Compression failed: error %zu\n", n); + goto cleanup; + } + + offset += n; + count_out += n; + if (size - offset < frame_size + LZ4_FOOTER_SIZE) { + printf("Writing %zu bytes\n", offset); + + k = fwrite(buf, 1, offset, out); + if (k < offset) { + if (ferror(out)) + printf("Write failed\n"); + else + printf("Short write\n"); + goto cleanup; + } + + offset = 0; + } + } + + n = LZ4F_compressEnd(ctx, buf + offset, size - offset, NULL); + if (LZ4F_isError(n)) { + printf("Failed to end compression: error %zu\n", n); + goto cleanup; + } + + offset += n; + count_out += n; + printf("Writing %zu bytes\n", offset); + + k = fwrite(buf, 1, offset, out); + if (k < offset) { + if (ferror(out)) + printf("Write failed\n"); + else + printf("Short write\n"); + goto cleanup; + } + + *size_in = count_in; + *size_out = count_out; + r = 0; cleanup: - if (ctx) - LZ4F_freeCompressionContext(ctx); - free(src); - free(buf); - return r; + if (ctx) + LZ4F_freeCompressionContext(ctx); + free(src); + free(buf); + return r; } static size_t get_block_size(const LZ4F_frameInfo_t* info) { - switch (info->blockSizeID) { + switch (info->blockSizeID) { case LZ4F_default: - case LZ4F_max64KB: return 1 << 16; - case LZ4F_max256KB: return 1 << 18; - case LZ4F_max1MB: return 1 << 20; - case LZ4F_max4MB: return 1 << 22; - default: - printf("Impossible unless more block sizes are allowed\n"); - exit(1); - } + case LZ4F_max64KB: return 1 << 16; + case LZ4F_max256KB: return 1 << 18; + case LZ4F_max1MB: return 1 << 20; + case LZ4F_max4MB: return 1 << 22; + default: + printf("Impossible unless more block sizes are allowed\n"); + exit(1); + } } static size_t decompress_file(FILE *in, FILE *out) { - void* const src = malloc(BUF_SIZE); - void* dst = NULL; - size_t dstCapacity = 0; - LZ4F_dctx *dctx = NULL; - size_t ret; + void* const src = malloc(BUF_SIZE); + void* dst = NULL; + size_t dstCapacity = 0; + LZ4F_dctx *dctx = NULL; + size_t ret; - /* Initialization */ + /* Initialization */ if (!src) { perror("decompress_file(src)"); goto cleanup; } - ret = LZ4F_createDecompressionContext(&dctx, 100); - if (LZ4F_isError(ret)) { - printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(ret)); - goto cleanup; - } - - /* Decompression */ - ret = 1; - while (ret != 0) { - /* Load more input */ - size_t srcSize = fread(src, 1, BUF_SIZE, in); - void* srcPtr = src; - void* srcEnd = srcPtr + srcSize; - if (srcSize == 0 || ferror(in)) { - printf("Decompress: not enough input or error reading file\n"); - goto cleanup; - } - /* Allocate destination buffer if it isn't already */ - if (!dst) { - LZ4F_frameInfo_t info; - ret = LZ4F_getFrameInfo(dctx, &info, src, &srcSize); - if (LZ4F_isError(ret)) { - printf("LZ4F_getFrameInfo error: %s\n", LZ4F_getErrorName(ret)); - goto cleanup; - } - /* Allocating enough space for an entire block isn't necessary for - * correctness, but it allows some memcpy's to be elided. - */ - dstCapacity = get_block_size(&info); - dst = malloc(dstCapacity); + ret = LZ4F_createDecompressionContext(&dctx, 100); + if (LZ4F_isError(ret)) { + printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(ret)); + goto cleanup; + } + + /* Decompression */ + ret = 1; + while (ret != 0) { + /* Load more input */ + size_t srcSize = fread(src, 1, BUF_SIZE, in); + void* srcPtr = src; + void* srcEnd = srcPtr + srcSize; + if (srcSize == 0 || ferror(in)) { + printf("Decompress: not enough input or error reading file\n"); + goto cleanup; + } + /* Allocate destination buffer if it isn't already */ + if (!dst) { + LZ4F_frameInfo_t info; + ret = LZ4F_getFrameInfo(dctx, &info, src, &srcSize); + if (LZ4F_isError(ret)) { + printf("LZ4F_getFrameInfo error: %s\n", LZ4F_getErrorName(ret)); + goto cleanup; + } + /* Allocating enough space for an entire block isn't necessary for + * correctness, but it allows some memcpy's to be elided. + */ + dstCapacity = get_block_size(&info); + dst = malloc(dstCapacity); if (!dst) { perror("decompress_file(dst)"); goto cleanup; } - srcPtr += srcSize; - srcSize = srcEnd - srcPtr; - } - /* Decompress: - * Continue while there is more input to read and the frame isn't over. - * If srcPtr == srcEnd then we know that there is no more output left in the - * internal buffer left to flush. - */ - while (srcPtr != srcEnd && ret != 0) { - /* INVARIANT: Any data left in dst has already been written */ - size_t dstSize = dstCapacity; - ret = LZ4F_decompress(dctx, dst, &dstSize, srcPtr, &srcSize, /* LZ4F_decompressOptions_t */ NULL); - if (LZ4F_isError(ret)) { - printf("Decompression error: %s\n", LZ4F_getErrorName(ret)); - goto cleanup; - } - /* Flush output */ - if (dstSize != 0){ - size_t written = fwrite(dst, 1, dstSize, out); - printf("Writing %zu bytes\n", dstSize); - if (written != dstSize) { - printf("Decompress: Failed to write to file\n"); - goto cleanup; - } - } - /* Update input */ - srcPtr += srcSize; - srcSize = srcEnd - srcPtr; - } - } - /* Check that there isn't trailing input data after the frame. - * It is valid to have multiple frames in the same file, but this example - * doesn't support it. - */ - ret = fread(src, 1, 1, in); - if (ret != 0 || !feof(in)) { - printf("Decompress: Trailing data left in file after frame\n"); - goto cleanup; - } + srcPtr += srcSize; + srcSize = srcEnd - srcPtr; + } + /* Decompress: + * Continue while there is more input to read and the frame isn't over. + * If srcPtr == srcEnd then we know that there is no more output left in the + * internal buffer left to flush. + */ + while (srcPtr != srcEnd && ret != 0) { + /* INVARIANT: Any data left in dst has already been written */ + size_t dstSize = dstCapacity; + ret = LZ4F_decompress(dctx, dst, &dstSize, srcPtr, &srcSize, /* LZ4F_decompressOptions_t */ NULL); + if (LZ4F_isError(ret)) { + printf("Decompression error: %s\n", LZ4F_getErrorName(ret)); + goto cleanup; + } + /* Flush output */ + if (dstSize != 0){ + size_t written = fwrite(dst, 1, dstSize, out); + printf("Writing %zu bytes\n", dstSize); + if (written != dstSize) { + printf("Decompress: Failed to write to file\n"); + goto cleanup; + } + } + /* Update input */ + srcPtr += srcSize; + srcSize = srcEnd - srcPtr; + } + } + /* Check that there isn't trailing input data after the frame. + * It is valid to have multiple frames in the same file, but this example + * doesn't support it. + */ + ret = fread(src, 1, 1, in); + if (ret != 0 || !feof(in)) { + printf("Decompress: Trailing data left in file after frame\n"); + goto cleanup; + } cleanup: - free(src); - free(dst); - return LZ4F_freeDecompressionContext(dctx); /* note : free works on NULL */ + free(src); + free(dst); + return LZ4F_freeDecompressionContext(dctx); /* note : free works on NULL */ } int compare(FILE* fp0, FILE* fp1) { - int result = 0; + int result = 0; - while(0 == result) { - char b0[1024]; - char b1[1024]; - const size_t r0 = fread(b0, 1, sizeof(b0), fp0); - const size_t r1 = fread(b1, 1, sizeof(b1), fp1); + while(0 == result) { + char b0[1024]; + char b1[1024]; + const size_t r0 = fread(b0, 1, sizeof(b0), fp0); + const size_t r1 = fread(b1, 1, sizeof(b1), fp1); - result = (int) r0 - (int) r1; + result = (int) r0 - (int) r1; - if (0 == r0 || 0 == r1) { - break; - } - if (0 == result) { - result = memcmp(b0, b1, r0); - } - } + if (0 == r0 || 0 == r1) { + break; + } + if (0 == result) { + result = memcmp(b0, b1, r0); + } + } - return result; + return result; } int main(int argc, const char **argv) { - char inpFilename[256] = { 0 }; - char lz4Filename[256] = { 0 }; - char decFilename[256] = { 0 }; - - if(argc < 2) { - printf("Please specify input filename\n"); - return 0; - } - - snprintf(inpFilename, 256, "%s", argv[1]); - snprintf(lz4Filename, 256, "%s.lz4", argv[1]); - snprintf(decFilename, 256, "%s.lz4.dec", argv[1]); - - printf("inp = [%s]\n", inpFilename); - printf("lz4 = [%s]\n", lz4Filename); - printf("dec = [%s]\n", decFilename); - - /* compress */ - { FILE* const inpFp = fopen(inpFilename, "rb"); - FILE* const outFp = fopen(lz4Filename, "wb"); - size_t sizeIn = 0; - size_t sizeOut = 0; - size_t ret; - - printf("compress : %s -> %s\n", inpFilename, lz4Filename); - ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut); - if (ret) { - printf("compress : failed with code %zu\n", ret); - return ret; - } - printf("%s: %zu → %zu bytes, %.1f%%\n", - inpFilename, sizeIn, sizeOut, - (double)sizeOut / sizeIn * 100); - printf("compress : done\n"); - - fclose(outFp); - fclose(inpFp); - } - - /* decompress */ - { FILE* const inpFp = fopen(lz4Filename, "rb"); - FILE* const outFp = fopen(decFilename, "wb"); - size_t ret; - - printf("decompress : %s -> %s\n", lz4Filename, decFilename); - ret = decompress_file(inpFp, outFp); - if (ret) { - printf("decompress : failed with code %zu\n", ret); - return ret; - } - printf("decompress : done\n"); - - fclose(outFp); - fclose(inpFp); - } - - /* verify */ - { FILE* const inpFp = fopen(inpFilename, "rb"); - FILE* const decFp = fopen(decFilename, "rb"); - - printf("verify : %s <-> %s\n", inpFilename, decFilename); - const int cmp = compare(inpFp, decFp); - if(0 == cmp) { - printf("verify : OK\n"); - } else { - printf("verify : NG\n"); - } - - fclose(decFp); - fclose(inpFp); - } + char inpFilename[256] = { 0 }; + char lz4Filename[256] = { 0 }; + char decFilename[256] = { 0 }; + + if(argc < 2) { + printf("Please specify input filename\n"); + return 0; + } + + snprintf(inpFilename, 256, "%s", argv[1]); + snprintf(lz4Filename, 256, "%s.lz4", argv[1]); + snprintf(decFilename, 256, "%s.lz4.dec", argv[1]); + + printf("inp = [%s]\n", inpFilename); + printf("lz4 = [%s]\n", lz4Filename); + printf("dec = [%s]\n", decFilename); + + /* compress */ + { FILE* const inpFp = fopen(inpFilename, "rb"); + FILE* const outFp = fopen(lz4Filename, "wb"); + size_t sizeIn = 0; + size_t sizeOut = 0; + size_t ret; + + printf("compress : %s -> %s\n", inpFilename, lz4Filename); + ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut); + if (ret) { + printf("compress : failed with code %zu\n", ret); + return ret; + } + printf("%s: %zu → %zu bytes, %.1f%%\n", + inpFilename, sizeIn, sizeOut, + (double)sizeOut / sizeIn * 100); + printf("compress : done\n"); + + fclose(outFp); + fclose(inpFp); + } + + /* decompress */ + { FILE* const inpFp = fopen(lz4Filename, "rb"); + FILE* const outFp = fopen(decFilename, "wb"); + size_t ret; + + printf("decompress : %s -> %s\n", lz4Filename, decFilename); + ret = decompress_file(inpFp, outFp); + if (ret) { + printf("decompress : failed with code %zu\n", ret); + return ret; + } + printf("decompress : done\n"); + + fclose(outFp); + fclose(inpFp); + } + + /* verify */ + { FILE* const inpFp = fopen(inpFilename, "rb"); + FILE* const decFp = fopen(decFilename, "rb"); + + printf("verify : %s <-> %s\n", inpFilename, decFilename); + const int cmp = compare(inpFp, decFp); + if(0 == cmp) { + printf("verify : OK\n"); + } else { + printf("verify : NG\n"); + } + + fclose(decFp); + fclose(inpFp); + } } @@ -945,6 +945,7 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream) int LZ4_freeStream (LZ4_stream_t* LZ4_stream) { + if (!LZ4_stream) return 0; /* support free on NULL */ FREEMEM(LZ4_stream); return (0); } @@ -1284,11 +1285,6 @@ int LZ4_decompress_fast(const char* source, char* dest, int originalSize) /*===== streaming decompression functions =====*/ -/* - * If you prefer dynamic allocation methods, - * LZ4_createStreamDecode() - * provides a pointer (void*) towards an initialized LZ4_streamDecode_t structure. - */ LZ4_streamDecode_t* LZ4_createStreamDecode(void) { LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t)); @@ -1297,6 +1293,7 @@ LZ4_streamDecode_t* LZ4_createStreamDecode(void) int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) { + if (!LZ4_stream) return 0; /* support free on NULL */ FREEMEM(LZ4_stream); return 0; } @@ -259,8 +259,8 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in * Important : Previous data blocks are assumed to remain present and unmodified ! * 'dst' buffer must be already allocated. * If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. - * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero. - * After an error, the stream status is invalid, and it can only be reset or freed. + * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function @return==0. + * After an error, the stream status is invalid, it can only be reset or freed. */ LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); @@ -291,22 +291,21 @@ LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_str */ LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); -/*! -LZ4_decompress_*_continue() : - These decoding functions allow decompression of multiple blocks in "streaming" mode. - Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) - In the case of a ring buffers, decoding buffer must be either : - - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) - In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). - - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. - maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. - In which case, encoding and decoding buffers do not need to be synchronized, - and encoding ring buffer can have any size, including small ones ( < 64 KB). - - _At least_ 64 KB + 8 bytes + maxBlockSize. - In which case, encoding and decoding buffers do not need to be synchronized, - and encoding ring buffer can have any size, including larger than decoding buffer. - Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, - and indicate where it is saved using LZ4_setStreamDecode() +/*! LZ4_decompress_*_continue() : + * These decoding functions allow decompression of multiple blocks in "streaming" mode. + * Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) + * In the case of a ring buffers, decoding buffer must be either : + * - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) + * In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). + * - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. + * maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. + * In which case, encoding and decoding buffers do not need to be synchronized, + * and encoding ring buffer can have any size, including small ones ( < 64 KB). + * - _At least_ 64 KB + 8 bytes + maxBlockSize. + * In which case, encoding and decoding buffers do not need to be synchronized, + * and encoding ring buffer can have any size, including larger than decoding buffer. + * Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, + * and indicate where it is saved using LZ4_setStreamDecode() */ LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); @@ -380,7 +379,7 @@ typedef struct { * init this structure before first use. * note : only use in association with static linking ! * this definition is not API/ABI safe, - * and may change in a future version ! + * it may change in a future version ! */ #define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) #define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) @@ -458,6 +457,7 @@ LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4 #endif /* LZ4_H_2983827168210 */ + #if defined (__cplusplus) } #endif diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 994cc8b..c27fc51 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -61,6 +61,7 @@ You can contact the author at : **************************************/ #include "lz4frame_static.h" #include "lz4.h" +#define LZ4_HC_STATIC_LINKING_ONLY #include "lz4hc.h" #define XXH_STATIC_LINKING_ONLY #include "xxhash.h" @@ -158,7 +159,7 @@ static void LZ4F_writeLE64 (void* dst, U64 value64) #define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB static const size_t minFHSize = 7; -static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 15 */ +static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 19 */ static const size_t BHSize = 4; @@ -170,6 +171,7 @@ typedef struct LZ4F_cctx_s LZ4F_preferences_t prefs; U32 version; U32 cStage; + const LZ4F_CDict* cdict; size_t maxBlockSize; size_t maxBufferSize; BYTE* tmpBuff; @@ -178,7 +180,7 @@ typedef struct LZ4F_cctx_s U64 totalInSize; XXH32_state_t xxh; void* lz4CtxPtr; - U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */ + U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */ } LZ4F_cctx_t; @@ -216,6 +218,8 @@ static LZ4F_errorCode_t err0r(LZ4F_errorCodes code) unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; } +int LZ4F_compressionLevel_max(void) { return LZ4HC_CLEVEL_MAX; } + /*-************************************ * Private functions @@ -256,7 +260,7 @@ static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSI return requestedBSID; } -/* LZ4F_compressBound() : +/* LZ4F_compressBound_internal() : * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations. * prefsPtr is optional : if NULL is provided, preferences will be set to cover worst case scenario. * Result is always the same for a srcSize and prefsPtr, so it can be relied upon to size reusable buffers. @@ -291,7 +295,7 @@ static size_t LZ4F_compressBound_internal(size_t srcSize, size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { LZ4F_preferences_t prefs; - size_t const headerSize = maxFHSize; /* max header size, including magic number and frame content size */ + size_t const headerSize = maxFHSize; /* max header size, including optional fields */ if (preferencesPtr!=NULL) prefs = *preferencesPtr; else memset(&prefs, 0, sizeof(prefs)); @@ -301,16 +305,19 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere } -/*! LZ4F_compressFrame() : - * Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.0, in a single step. - * The most important rule is that dstBuffer MUST be large enough (dstCapacity) to ensure compression completion even in worst case. - * If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode) - * Get the minimum value of dstCapacity by using LZ4F_compressFrameBound(). - * The LZ4F_preferences_t structure is optional : if NULL is provided as argument, preferences will be set to default. - * The result of the function is the number of bytes written into dstBuffer. - * The function outputs an error code if it fails (can be tested using LZ4F_isError()) +/*! LZ4F_compressFrame_usingCDict() : + * Compress srcBuffer using a dictionary, in a single step. + * cdict can be NULL, in which case, no dictionary is used. + * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). + * The LZ4F_preferences_t structure is optional : you may provide NULL as argument, + * however, it's the only way to provide a dictID, so it's not recommended. + * @return : number of bytes written into dstBuffer, + * or an error code if it fails (can be tested using LZ4F_isError()) */ -size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr) +size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* preferencesPtr) { LZ4F_cctx_t cctxI; LZ4_stream_t lz4ctx; @@ -321,10 +328,8 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu BYTE* const dstEnd = dstStart + dstCapacity; memset(&cctxI, 0, sizeof(cctxI)); - memset(&options, 0, sizeof(options)); - cctxI.version = LZ4F_VERSION; - cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works because autoflush==1 & stableSrc==1 */ + cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */ if (preferencesPtr!=NULL) prefs = *preferencesPtr; @@ -333,22 +338,23 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu if (prefs.frameInfo.contentSize != 0) prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */ - if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { - cctxI.lz4CtxPtr = &lz4ctx; - cctxI.lz4CtxLevel = 1; - } /* otherwise : will be created within LZ4F_compressBegin */ - prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); prefs.autoFlush = 1; if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID)) - prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* no need for linked blocks */ + prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* only one block => no need for inter-block link */ + if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { + cctxI.lz4CtxPtr = &lz4ctx; + cctxI.lz4CtxLevel = 1; + } /* fast compression context pre-created on stack */ + + memset(&options, 0, sizeof(options)); options.stableSrc = 1; - if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs)) /* condition to guarantee success */ + if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs)) /* condition to guarantee success */ return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); - { size_t const headerSize = LZ4F_compressBegin(&cctxI, dstBuffer, dstCapacity, &prefs); /* write header */ + { size_t const headerSize = LZ4F_compressBegin_usingCDict(&cctxI, dstBuffer, dstCapacity, cdict, &prefs); /* write header */ if (LZ4F_isError(headerSize)) return headerSize; dstPtr += headerSize; /* header size */ } @@ -360,24 +366,91 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu if (LZ4F_isError(tailSize)) return tailSize; dstPtr += tailSize; } - if (prefs.compressionLevel >= LZ4HC_CLEVEL_MIN) /* Ctx allocation only for lz4hc */ + if (prefs.compressionLevel >= LZ4HC_CLEVEL_MIN) /* Ctx allocation only for lz4hc */ FREEMEM(cctxI.lz4CtxPtr); return (dstPtr - dstStart); } +/*! LZ4F_compressFrame() : + * Compress an entire srcBuffer into a valid LZ4 frame, in a single step. + * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). + * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. + * @return : number of bytes written into dstBuffer. + * or an error code if it fails (can be tested using LZ4F_isError()) + */ +size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_preferences_t* preferencesPtr) +{ + return LZ4F_compressFrame_usingCDict(dstBuffer, dstCapacity, + srcBuffer, srcSize, + NULL, preferencesPtr); +} + + +/*-*************************************************** +* Dictionary compression +*****************************************************/ + +struct LZ4F_CDict_s { + void* dictContent; + LZ4_stream_t* fastCtx; + LZ4_streamHC_t* HCCtx; +}; /* typedef'd to LZ4F_CDict within lz4frame_static.h */ + +/*! LZ4_createCDict() : + * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. + * LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. + * LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict + * @return : digested dictionary for compression, or NULL if failed */ +LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize) +{ + const char* dictStart = (const char*)dictBuffer; + LZ4F_CDict* cdict = (LZ4F_CDict*) malloc(sizeof(*cdict)); + if (!cdict) return NULL; + if (dictSize > 64 KB) { + dictStart += dictSize - 64 KB; + dictSize = 64 KB; + } + cdict->dictContent = ALLOCATOR(dictSize); + cdict->fastCtx = LZ4_createStream(); + cdict->HCCtx = LZ4_createStreamHC(); + if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) { + LZ4F_freeCDict(cdict); + return NULL; + } + memcpy(cdict->dictContent, dictStart, dictSize); + LZ4_resetStream(cdict->fastCtx); + LZ4_loadDict (cdict->fastCtx, (const char*)cdict->dictContent, (int)dictSize); + LZ4_resetStreamHC(cdict->HCCtx, LZ4HC_CLEVEL_DEFAULT); + LZ4_loadDictHC(cdict->HCCtx, (const char*)cdict->dictContent, (int)dictSize); + return cdict; +} + +void LZ4F_freeCDict(LZ4F_CDict* cdict) +{ + if (cdict==NULL) return; /* support free on NULL */ + FREEMEM(cdict->dictContent); + LZ4_freeStream(cdict->fastCtx); + LZ4_freeStreamHC(cdict->HCCtx); + FREEMEM(cdict); +} + + /*-********************************* * Advanced compression functions ***********************************/ /*! LZ4F_createCompressionContext() : - * The first thing to do is to create a compressionContext object, which will be used in all compression operations. - * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. - * The version provided MUST be LZ4F_VERSION. It is intended to track potential version differences between different binaries. - * The function will provide a pointer to an allocated LZ4F_compressionContext_t object. - * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. - * Object can release its memory using LZ4F_freeCompressionContext(); + * The first thing to do is to create a compressionContext object, which will be used in all compression operations. + * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. + * The version provided MUST be LZ4F_VERSION. It is intended to track potential incompatible differences between different binaries. + * The function will provide a pointer to an allocated LZ4F_compressionContext_t object. + * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. + * Object can release its memory using LZ4F_freeCompressionContext(); */ LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version) { @@ -385,7 +458,7 @@ LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_c if (cctxPtr==NULL) return err0r(LZ4F_ERROR_allocation_failed); cctxPtr->version = version; - cctxPtr->cStage = 0; /* Next stage : write header */ + cctxPtr->cStage = 0; /* Next stage : init stream */ *LZ4F_compressionContextPtr = (LZ4F_compressionContext_t)cctxPtr; @@ -397,8 +470,8 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp { LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)LZ4F_compressionContext; - if (cctxPtr != NULL) { /* null pointers can be safely provided to this function, like free() */ - FREEMEM(cctxPtr->lz4CtxPtr); + if (cctxPtr != NULL) { /* support free on NULL */ + FREEMEM(cctxPtr->lz4CtxPtr); /* works because LZ4_streamHC_t and LZ4_stream_t are simple POD types */ FREEMEM(cctxPtr->tmpBuff); FREEMEM(LZ4F_compressionContext); } @@ -407,22 +480,23 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp } -/*! LZ4F_compressBegin() : - * will write the frame header into dstBuffer. - * dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_HEADER_SIZE_MAX bytes. - * @return : number of bytes written into dstBuffer for the header - * or an error code (can be tested using LZ4F_isError()) +/*! LZ4F_compressBegin_usingCDict() : + * init streaming compression and writes frame header into dstBuffer. + * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes. + * @return : number of bytes written into dstBuffer for the header + * or an error code (can be tested using LZ4F_isError()) */ -size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* preferencesPtr) +size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* preferencesPtr) { LZ4F_preferences_t prefNull; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; BYTE* headerStart; - size_t requiredBuffSize; if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); - if (cctxPtr->cStage != 0) return err0r(LZ4F_ERROR_GENERIC); memset(&prefNull, 0, sizeof(prefNull)); if (preferencesPtr == NULL) preferencesPtr = &prefNull; cctxPtr->prefs = *preferencesPtr; @@ -437,31 +511,46 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC(); if (cctxPtr->lz4CtxPtr == NULL) return err0r(LZ4F_ERROR_allocation_failed); cctxPtr->lz4CtxLevel = tableID; - } - } + } } /* Buffer Management */ - if (cctxPtr->prefs.frameInfo.blockSizeID == 0) cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; + if (cctxPtr->prefs.frameInfo.blockSizeID == 0) + cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; cctxPtr->maxBlockSize = LZ4F_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID); - requiredBuffSize = cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 128 KB); - if (preferencesPtr->autoFlush) - requiredBuffSize = (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 64 KB; /* just needs dict */ + { size_t const requiredBuffSize = preferencesPtr->autoFlush ? + (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 64 KB : /* only needs windows size */ + cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 128 KB); - if (cctxPtr->maxBufferSize < requiredBuffSize) { - cctxPtr->maxBufferSize = 0; - FREEMEM(cctxPtr->tmpBuff); - cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize); - if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed); - cctxPtr->maxBufferSize = requiredBuffSize; - } + if (cctxPtr->maxBufferSize < requiredBuffSize) { + cctxPtr->maxBufferSize = 0; + FREEMEM(cctxPtr->tmpBuff); + cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize); + if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed); + cctxPtr->maxBufferSize = requiredBuffSize; + } } cctxPtr->tmpIn = cctxPtr->tmpBuff; cctxPtr->tmpInSize = 0; XXH32_reset(&(cctxPtr->xxh), 0); - if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) - LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr)); - else - LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel); + + /* context init */ + cctxPtr->cdict = cdict; + if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) { + /* frame init only for blockLinked : blockIndependent will be init at each block */ + if (cdict) { + if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { + memcpy(cctxPtr->lz4CtxPtr, cdict->fastCtx, sizeof(*cdict->fastCtx)); + } else { + memcpy(cctxPtr->lz4CtxPtr, cdict->HCCtx, sizeof(*cdict->HCCtx)); + LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + } + } else { + if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) + LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr)); + else + LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel); + } + } /* Magic Number */ LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER); @@ -470,9 +559,10 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit /* FLG Byte */ *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */ - + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) /* Block mode */ + + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) /* Block mode */ + + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3) /* Frame content size */ + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) /* Frame checksum */ - + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3)); /* Frame content size */ + + (cctxPtr->prefs.frameInfo.dictID > 0) ); /* Dictionary ID */ /* BD Byte */ *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4); /* Optional Frame content size field */ @@ -481,16 +571,36 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit dstPtr += 8; cctxPtr->totalInSize = 0; } - /* CRC Byte */ + /* Optional dictionary ID field */ + if (cctxPtr->prefs.frameInfo.dictID) { + LZ4F_writeLE32(dstPtr, cctxPtr->prefs.frameInfo.dictID); + dstPtr += 4; + } + /* Header CRC Byte */ *dstPtr = LZ4F_headerChecksum(headerStart, dstPtr - headerStart); dstPtr++; cctxPtr->cStage = 1; /* header written, now request input data block */ - return (dstPtr - dstStart); } +/*! LZ4F_compressBegin() : + * init streaming compression and writes frame header into dstBuffer. + * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes. + * preferencesPtr can be NULL, in which case default parameters are selected. + * @return : number of bytes written into dstBuffer for the header + * or an error code (can be tested using LZ4F_isError()) + */ +size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const LZ4F_preferences_t* preferencesPtr) +{ + return LZ4F_compressBegin_usingCDict(cctxPtr, dstBuffer, dstCapacity, + NULL, preferencesPtr); +} + + /* LZ4F_compressBound() : * @ return size of Dst buffer given a srcSize to handle worst case situations. * The LZ4F_frameInfo_t structure is optional : if NULL, preferences will be set to cover worst case situations. @@ -502,13 +612,15 @@ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesP } -typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level); +typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level, const LZ4F_CDict* cdict); -static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx, int level) +static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx, int level, const LZ4F_CDict* cdict) { /* compress a single block */ BYTE* const cSizePtr = (BYTE*)dst; - U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1), level); + U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), + (int)(srcSize), (int)(srcSize-1), + level, cdict); LZ4F_writeLE32(cSizePtr, cSize); if (cSize == 0) { /* compression failed */ cSize = (U32)srcSize; @@ -519,32 +631,47 @@ static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, com } -static int LZ4F_localLZ4_compress_limitedOutput_withState(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level) +static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) { int const acceleration = (level < -1) ? -level : 1; + if (cdict) { + memcpy(ctx, cdict->fastCtx, sizeof(*cdict->fastCtx)); + return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration); + } return LZ4_compress_fast_extState(ctx, src, dst, srcSize, dstCapacity, acceleration); } -static int LZ4F_localLZ4_compress_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level) +static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) { int const acceleration = (level < -1) ? -level : 1; + (void)cdict; /* init once at beginning of frame */ return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration); } -static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) +static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) +{ + if (cdict) { + memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx)); + LZ4_setCompressionLevel((LZ4_streamHC_t*)ctx, level); + return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity); + } + return LZ4_compress_HC_extStateHC(ctx, src, dst, srcSize, dstCapacity, level); +} + +static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) { - (void) level; - return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstSize); + (void)level; (void)cdict; /* init once at beginning of frame */ + return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity); } static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level) { if (level < LZ4HC_CLEVEL_MIN) { - if (blockMode == LZ4F_blockIndependent) return LZ4F_localLZ4_compress_limitedOutput_withState; - return LZ4F_localLZ4_compress_limitedOutput_continue; + if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlock; + return LZ4F_compressBlock_continue; } - if (blockMode == LZ4F_blockIndependent) return LZ4_compress_HC_extStateHC; - return LZ4F_localLZ4_compressHC_limitedOutput_continue; + if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlockHC; + return LZ4F_compressBlockHC_continue; } static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr) @@ -557,14 +684,12 @@ static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr) typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus; /*! LZ4F_compressUpdate() : -* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. -* The most important rule is that dstBuffer MUST be large enough (dstCapacity) to ensure compression completion even in worst case. -* If this condition is not respected, LZ4F_compress() will fail (result is an errorCode) -* You can get the minimum value of dstCapacity by using LZ4F_compressBound() -* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. -* The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered. -* The function outputs an error code if it fails (can be tested using LZ4F_isError()) -*/ + * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. + * dstBuffer MUST be >= LZ4F_compressBound(srcSize, preferencesPtr). + * LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered. + * or an error code if it fails (which can be tested using LZ4F_isError()) + */ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr) { LZ4F_compressOptions_t cOptionsNull; @@ -597,7 +722,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy); srcPtr += sizeToCopy; - dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict); if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize; cctxPtr->tmpInSize = 0; @@ -605,16 +730,16 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci } while ((size_t)(srcEnd - srcPtr) >= blockSize) { - /* compress full block */ + /* compress full blocks */ lastBlockCompressed = fromSrcBuffer; - dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict); srcPtr += blockSize; } if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) { /* compress remaining input < blockSize */ lastBlockCompressed = fromSrcBuffer; - dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict); srcPtr = srcEnd; } @@ -676,7 +801,7 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); /* compress tmp buffer */ - dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict); if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize; cctxPtr->tmpInSize = 0; @@ -801,6 +926,8 @@ typedef enum { void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx) { dctx->dStage = dstage_getHeader; + dctx->dict = NULL; + dctx->dictSize = 0; } @@ -817,12 +944,14 @@ static size_t LZ4F_headerSize(const void* src, size_t srcSize) if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) return 8; /* control magic number */ - if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER) return err0r(LZ4F_ERROR_frameType_unknown); + if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER) + return err0r(LZ4F_ERROR_frameType_unknown); /* Frame Header Size */ { BYTE const FLG = ((const BYTE*)src)[4]; U32 const contentSizeFlag = (FLG>>3) & _1BIT; - return contentSizeFlag ? maxFHSize : minFHSize; + U32 const dictIDFlag = FLG & _1BIT; + return minFHSize + (contentSizeFlag*8) + (dictIDFlag*4); } } @@ -837,7 +966,7 @@ static size_t LZ4F_headerSize(const void* src, size_t srcSize) */ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcSize) { - unsigned blockMode, contentSizeFlag, contentChecksumFlag, blockSizeID; + unsigned blockMode, contentSizeFlag, contentChecksumFlag, dictIDFlag, blockSizeID; size_t frameHeaderSize; const BYTE* srcPtr = (const BYTE*)src; @@ -860,7 +989,8 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcS } /* control magic number */ - if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) return err0r(LZ4F_ERROR_frameType_unknown); + if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) + return err0r(LZ4F_ERROR_frameType_unknown); dctxPtr->frameInfo.frameType = LZ4F_frame; /* Flags */ @@ -870,14 +1000,15 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcS blockMode = (FLG>>5) & _1BIT; contentSizeFlag = (FLG>>3) & _1BIT; contentChecksumFlag = (FLG>>2) & _1BIT; + dictIDFlag = FLG & _1BIT; /* validate */ - if (((FLG>>0)&_2BITS) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bits */ + if (((FLG>>1)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bits */ if (version != 1) return err0r(LZ4F_ERROR_headerVersion_wrong); /* Version Number, only supported value */ if (blockChecksumFlag != 0) return err0r(LZ4F_ERROR_blockChecksum_unsupported); /* Not supported for the time being */ } /* Frame Header Size */ - frameHeaderSize = contentSizeFlag ? maxFHSize : minFHSize; + frameHeaderSize = minFHSize + (contentSizeFlag*8) + (dictIDFlag*4); if (srcSize < frameHeaderSize) { /* not enough input to fully decode frame header */ @@ -898,8 +1029,10 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcS } /* check header */ - { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5); - if (HC != srcPtr[frameHeaderSize-1]) return err0r(LZ4F_ERROR_headerChecksum_invalid); } + { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5); + if (HC != srcPtr[frameHeaderSize-1]) + return err0r(LZ4F_ERROR_headerChecksum_invalid); + } /* save */ dctxPtr->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode; @@ -907,7 +1040,10 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcS dctxPtr->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID; dctxPtr->maxBlockSize = LZ4F_getBlockSize(blockSizeID); if (contentSizeFlag) - dctxPtr->frameRemainingSize = dctxPtr->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6); + dctxPtr->frameRemainingSize = + dctxPtr->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6); + if (dictIDFlag) + dctxPtr->frameInfo.dictID = LZ4F_readLE32(srcPtr + frameHeaderSize - 5); dctxPtr->dStage = dstage_init; @@ -967,19 +1103,6 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctxPtr, LZ4F_frameInfo_t* frameIn } -/* trivial redirector, for common prototype */ -static int LZ4F_decompress_safe (const char* src, - char* dst, - int compressedSize, - int dstCapacity, - const char* dictStart, - int dictSize) -{ - (void)dictStart; (void)dictSize; - return LZ4_decompress_safe (src, dst, compressedSize, dstCapacity); -} - - static void LZ4F_updateDict(LZ4F_dctx* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp) { if (dctxPtr->dictSize==0) @@ -1040,21 +1163,22 @@ static void LZ4F_updateDict(LZ4F_dctx* dctxPtr, const BYTE* dstPtr, size_t dstSi /*! LZ4F_decompress() : - * Call this function repetitively to regenerate data compressed within srcBuffer. - * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr. + * Call this function repetitively to regenerate compressed data in srcBuffer. + * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer + * into dstBuffer of capacity *dstSizePtr. * - * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). + * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). * - * The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). - * If the number of bytes read is < number of bytes provided, then the decompression operation is not complete. - * Remaining data will have to be presented again in a subsequent invocation. + * The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). + * If number of bytes read is < number of bytes provided, then decompression operation is not complete. + * Remaining data will have to be presented again in a subsequent invocation. * - * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress. - * Basically, it's the size of the current (or remaining) compressed block + header of next block. - * Respecting the hint provides some boost to performance, since it allows less buffer shuffling. - * Note that this is just a hint, it's always possible to any srcSize value. - * When a frame is fully decoded, @return will be 0. - * If decompression failed, @return is an error code which can be tested using LZ4F_isError(). + * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress. + * Schematically, it's the size of the current (or remaining) compressed block + header of next block. + * Respecting the hint provides a small boost to performance, since it allows less buffer shuffling. + * Note that this is just a hint, and it's always possible to any srcSize value. + * When a frame is fully decoded, @return will be 0. + * If decompression failed, @return is an error code which can be tested using LZ4F_isError(). */ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, void* dstBuffer, size_t* dstSizePtr, @@ -1131,8 +1255,6 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, } } dctxPtr->tmpInSize = 0; dctxPtr->tmpInTarget = 0; - dctxPtr->dict = dctxPtr->tmpOutBuffer; - dctxPtr->dictSize = 0; dctxPtr->tmpOut = dctxPtr->tmpOutBuffer; dctxPtr->tmpOutStart = 0; dctxPtr->tmpOutSize = 0; @@ -1247,15 +1369,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, break; case dstage_decodeCBlock_intoDst: - { int (*decoder)(const char*, char*, int, int, const char*, int); - int decodedSize; - - if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked) - decoder = LZ4_decompress_safe_usingDict; - else - decoder = LZ4F_decompress_safe; - - decodedSize = decoder((const char*)selectedIn, (char*)dstPtr, + { int const decodedSize = LZ4_decompress_safe_usingDict( + (const char*)selectedIn, (char*)dstPtr, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); if (decodedSize < 0) return err0r(LZ4F_ERROR_GENERIC); /* decompression failed */ @@ -1275,13 +1390,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, case dstage_decodeCBlock_intoTmp: /* not enough place into dst : decode into tmpOut */ - { int (*decoder)(const char*, char*, int, int, const char*, int); - int decodedSize; - - if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked) - decoder = LZ4_decompress_safe_usingDict; - else - decoder = LZ4F_decompress_safe; + { int decodedSize; /* ensure enough place for tmpOut */ if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked) { @@ -1299,7 +1408,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, } /* Decode */ - decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut, + decodedSize = LZ4_decompress_safe_usingDict( + (const char*)selectedIn, (char*)dctxPtr->tmpOut, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); if (decodedSize < 0) @@ -1342,7 +1452,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, return err0r(LZ4F_ERROR_frameSize_wrong); /* incorrect frame size decoded */ if (suffixSize == 0) { /* frame completed */ nextSrcSizeHint = 0; - dctxPtr->dStage = dstage_getHeader; + LZ4F_resetDecompressionContext(dctxPtr); doAnotherStage = 0; break; } @@ -1376,7 +1486,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, U32 const resultCRC = XXH32_digest(&(dctxPtr->xxh)); if (readCRC != resultCRC) return err0r(LZ4F_ERROR_contentChecksum_invalid); nextSrcSizeHint = 0; - dctxPtr->dStage = dstage_getHeader; + LZ4F_resetDecompressionContext(dctxPtr); doAnotherStage = 0; break; } @@ -1396,7 +1506,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, case dstage_storeSFrameSize: { size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; - if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; + if (sizeToCopy > (size_t)(srcEnd - srcPtr)) + sizeToCopy = srcEnd - srcPtr; memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy); srcPtr += sizeToCopy; dctxPtr->tmpInSize += sizeToCopy; @@ -1419,24 +1530,25 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, case dstage_skipSkippable: { size_t skipSize = dctxPtr->tmpInTarget; - if (skipSize > (size_t)(srcEnd-srcPtr)) skipSize = srcEnd-srcPtr; + if (skipSize > (size_t)(srcEnd-srcPtr)) + skipSize = srcEnd-srcPtr; srcPtr += skipSize; dctxPtr->tmpInTarget -= skipSize; doAnotherStage = 0; nextSrcSizeHint = dctxPtr->tmpInTarget; if (nextSrcSizeHint) break; - dctxPtr->dStage = dstage_getHeader; + LZ4F_resetDecompressionContext(dctxPtr); break; } } } - /* preserve dictionary within tmp if necessary */ + /* preserve history within tmp if necessary */ if ( (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) - &&(dctxPtr->dict != dctxPtr->tmpOutBuffer) - &&(!decompressOptionsPtr->stableDst) - &&((unsigned)(dctxPtr->dStage-1) < (unsigned)(dstage_getSuffix-1)) - ) + && (dctxPtr->dict != dctxPtr->tmpOutBuffer) + && (dctxPtr->dStage != dstage_getHeader) + && (!decompressOptionsPtr->stableDst) + && ((unsigned)(dctxPtr->dStage-1) < (unsigned)(dstage_getSuffix-1)) ) { if (dctxPtr->dStage == dstage_flushOut) { size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; @@ -1466,3 +1578,23 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr, *dstSizePtr = (dstPtr - dstStart); return nextSrcSizeHint; } + +/*! LZ4F_decompress_usingDict() : + * Same as LZ4F_decompress(), using a predefined dictionary. + * Dictionary is used "in place", without any preprocessing. + * It must remain accessible throughout the entire frame decoding. + */ +size_t LZ4F_decompress_usingDict(LZ4F_dctx* dctxPtr, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const void* dict, size_t dictSize, + const LZ4F_decompressOptions_t* decompressOptionsPtr) +{ + if (dctxPtr->dStage <= dstage_init) { + dctxPtr->dict = (const BYTE*)dict; + dctxPtr->dictSize = dictSize; + } + return LZ4F_decompress(dctxPtr, dstBuffer, dstSizePtr, + srcBuffer, srcSizePtr, + decompressOptionsPtr); +} diff --git a/lib/lz4frame.h b/lib/lz4frame.h index dd2be58..844255b 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -100,8 +100,8 @@ LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return /*-************************************ * Frame compression types **************************************/ -/* #define LZ4F_DISABLE_OBSOLETE_ENUMS */ /* uncomment to disable obsolete enums */ -#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS +/* #define LZ4F_ENABLE_OBSOLETE_ENUMS // uncomment to enable obsolete enums */ +#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS # define LZ4F_OBSOLETE_ENUM(x) , LZ4F_DEPRECATE(x) = LZ4F_##x #else # define LZ4F_OBSOLETE_ENUM(x) @@ -145,7 +145,7 @@ typedef enum { LZ4F_OBSOLETE_ENUM(skippableFrame) } LZ4F_frameType_t; -#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS +#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS typedef LZ4F_blockSizeID_t blockSizeID_t; typedef LZ4F_blockMode_t blockMode_t; typedef LZ4F_frameType_t frameType_t; @@ -157,12 +157,13 @@ typedef LZ4F_contentChecksum_t contentChecksum_t; * It's not required to set all fields, as long as the structure was initially memset() to zero. * All reserved fields must be set to zero. */ typedef struct { - LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ - LZ4F_blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */ - LZ4F_contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */ - LZ4F_frameType_t frameType; /* LZ4F_frame, skippableFrame ; 0 == default */ - unsigned long long contentSize; /* Size of uncompressed (original) content ; 0 == unknown */ - unsigned reserved[2]; /* must be zero for forward compatibility */ + LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ + LZ4F_blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */ + LZ4F_contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */ + LZ4F_frameType_t frameType; /* LZ4F_frame, skippableFrame ; 0 == default */ + unsigned long long contentSize; /* Size of uncompressed content ; 0 == unknown */ + unsigned dictID; /* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */ + unsigned reserved[1]; /* must be zero for forward compatibility */ } LZ4F_frameInfo_t; /*! LZ4F_preferences_t : @@ -172,31 +173,32 @@ typedef struct { typedef struct { LZ4F_frameInfo_t frameInfo; int compressionLevel; /* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */ - unsigned autoFlush; /* 1 == always flush (reduce usage of tmp buffer) */ + unsigned autoFlush; /* 1 == always flush, to reduce usage of internal buffers */ unsigned reserved[4]; /* must be zero for forward compatibility */ } LZ4F_preferences_t; +LZ4FLIB_API int LZ4F_compressionLevel_max(void); + /*-********************************* * Simple compression function ***********************************/ -/*!LZ4F_compressFrameBound() : - * Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences. - * Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression. +/*! LZ4F_compressFrameBound() : + * Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences. + * Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression. */ LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr); -/*!LZ4F_compressFrame() : - * Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.1 - * An important rule is that dstBuffer MUST be large enough (dstCapacity) to store the result in worst case situation. - * This value is supplied by LZ4F_compressFrameBound(). - * If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode). - * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. +/*! LZ4F_compressFrame() : + * Compress an entire srcBuffer into a valid LZ4 frame. + * dstCapacity MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). + * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. * @return : number of bytes written into dstBuffer. * or an error code if it fails (can be tested using LZ4F_isError()) */ -LZ4FLIB_API size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr); - +LZ4FLIB_API size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_preferences_t* preferencesPtr); /*-*********************************** @@ -228,15 +230,17 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); /*---- Compression ----*/ -#define LZ4F_HEADER_SIZE_MAX 15 +#define LZ4F_HEADER_SIZE_MAX 19 /*! LZ4F_compressBegin() : - * will write the frame header into dstBuffer. - * dstCapacity must be large enough to store the header. Maximum header size is LZ4F_HEADER_SIZE_MAX bytes. + * will write the frame header into dstBuffer. + * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. * `prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default. * @return : number of bytes written into dstBuffer for the header * or an error code (which can be tested using LZ4F_isError()) */ -LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* prefsPtr); +LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_preferences_t* prefsPtr); /*! LZ4F_compressBound() : * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations. @@ -269,13 +273,13 @@ LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr); /*! LZ4F_compressEnd() : - * To properly finish an LZ4 frame, invoke LZ4F_compressEnd(). - * It will flush whatever data remained within `cctx` (like LZ4_flush()) - * and properly finalize the frame, with an endMark and a checksum. + * To properly finish an LZ4 frame, invoke LZ4F_compressEnd(). + * It will flush whatever data remained within `cctx` (like LZ4_flush()) + * and properly finalize the frame, with an endMark and a checksum. * `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default. * @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled) * or an error code if it fails (which can be tested using LZ4F_isError()) - * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task. + * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task. */ LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr); @@ -288,7 +292,7 @@ typedef LZ4F_dctx* LZ4F_decompressionContext_t; /* compatibility with previous typedef struct { unsigned stableDst; /* pledge that at least 64KB+64Bytes of previously decompressed data remain unmodifed where it was decoded. This optimization skips storage operations in tmp buffers */ - unsigned reserved[3]; + unsigned reserved[3]; /* must be set to zero for forward compatibility */ } LZ4F_decompressOptions_t; @@ -312,10 +316,10 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx); *************************************/ /*! LZ4F_getFrameInfo() : - * This function extracts frame parameters (such as max blockSize, frame checksum, etc.). - * Its usage is optional. - * Extracted information can typically be useful for allocation purposes. - * This function works in 2 situations : + * This function extracts frame parameters (max blockSize, dictID, etc.). + * Its usage is optional. + * Extracted information is typically useful for allocation and dictionary. + * This function works in 2 situations : * - At the beginning of a new frame, in which case * it will decode information from `srcBuffer`, starting the decoding process. * Input size must be large enough to successfully decode the entire frame header. @@ -323,41 +327,42 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx); * It's allowed to provide more input data than this minimum. * - After decoding has been started. * In which case, no input is read, frame parameters are extracted from dctx. - * - If decoding has barely started, but not yet extracted information from header, LZ4F_getFrameInfo() will fail. - * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). - * Decompression must resume from (srcBuffer + *srcSizePtr). + * - If decoding has barely started, but not yet extracted information from header, + * LZ4F_getFrameInfo() will fail. + * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). + * Decompression must resume from (srcBuffer + *srcSizePtr). * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call, - * or an error code which can be tested using LZ4F_isError() - * note 1 : in case of error, dctx is not modified. Decoding operation can resume safely. - * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure. + * or an error code which can be tested using LZ4F_isError(). + * note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely. + * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure. */ LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoPtr, const void* srcBuffer, size_t* srcSizePtr); /*! LZ4F_decompress() : - * Call this function repetitively to regenerate compressed data from `srcBuffer`. - * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr. + * Call this function repetitively to regenerate compressed data from `srcBuffer`. + * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr. * - * The number of bytes regenerated into dstBuffer is provided within *dstSizePtr (necessarily <= original value). + * The number of bytes regenerated into dstBuffer is provided within *dstSizePtr (necessarily <= original value). * - * The number of bytes consumed from srcBuffer is provided within *srcSizePtr (necessarily <= original value). - * Number of bytes consumed can be < number of bytes provided. - * It typically happens when dstBuffer is not large enough to contain all decoded data. - * Unconsumed source data must be presented again in subsequent invocations. + * The number of bytes consumed from srcBuffer is provided within *srcSizePtr (necessarily <= original value). + * Number of bytes consumed can be < number of bytes provided. + * It typically happens when dstBuffer is not large enough to contain all decoded data. + * Unconsumed source data must be presented again in subsequent invocations. * * `dstBuffer` content is expected to be flushed between each invocation, as its content will be overwritten. * `dstBuffer` itself can be changed at will between each consecutive function invocation. * - * @return is an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call. - * Schematically, it's the size of the current (or remaining) compressed block + header of next block. - * Respecting the hint provides some small speed benefit, because it skips intermediate buffers. - * This is just a hint though, it's always possible to provide any srcSize. - * When a frame is fully decoded, @return will be 0 (no more data expected). - * If decompression failed, @return is an error code, which can be tested using LZ4F_isError(). + * @return : an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call. + * Schematically, it's the size of the current (or remaining) compressed block + header of next block. + * Respecting the hint provides some small speed benefit, because it skips intermediate buffers. + * This is just a hint though, it's always possible to provide any srcSize. + * When a frame is fully decoded, @return will be 0 (no more data expected). + * If decompression failed, @return is an error code, which can be tested using LZ4F_isError(). * - * After a frame is fully decoded, dctx can be used again to decompress another frame. - * After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state. + * After a frame is fully decoded, dctx can be used again to decompress another frame. + * After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state. */ LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx, void* dstBuffer, size_t* dstSizePtr, diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h index d3bae82..b585b72 100644 --- a/lib/lz4frame_static.h +++ b/lib/lz4frame_static.h @@ -82,6 +82,60 @@ typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult); + +/********************************** + * Bulk processing dictionary API + *********************************/ +typedef struct LZ4F_CDict_s LZ4F_CDict; + +/*! LZ4_createCDict() : + * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. + * LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. + * LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict */ +LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize); +void LZ4F_freeCDict(LZ4F_CDict* CDict); + + +/*! LZ4_compressFrame_usingCDict() : + * Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary. + * If cdict==NULL, compress without a dictionary. + * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). + * If this condition is not respected, function will fail (@return an errorCode). + * The LZ4F_preferences_t structure is optional : you may provide NULL as argument, + * but it's not recommended, as it's the only way to provide dictID in the frame header. + * @return : number of bytes written into dstBuffer. + * or an error code if it fails (can be tested using LZ4F_isError()) */ +size_t LZ4F_compressFrame_usingCDict(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* preferencesPtr); + + +/*! LZ4F_compressBegin_usingCDict() : + * Inits streaming dictionary compression, and writes the frame header into dstBuffer. + * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. + * `prefsPtr` is optional : you may provide NULL as argument, + * however, it's the only way to provide dictID in the frame header. + * @return : number of bytes written into dstBuffer for the header, + * or an error code (which can be tested using LZ4F_isError()) */ +size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* prefsPtr); + + +/*! LZ4F_decompress_usingDict() : + * Same as LZ4F_decompress(), using a predefined dictionary. + * Dictionary is used "in place", without any preprocessing. + * It must remain accessible throughout the entire frame decoding. */ +size_t LZ4F_decompress_usingDict(LZ4F_dctx* dctxPtr, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const void* dict, size_t dictSize, + const LZ4F_decompressOptions_t* decompressOptionsPtr); + + #if defined (__cplusplus) } #endif diff --git a/lib/lz4hc.c b/lib/lz4hc.c index ca9c2e6..22eb071 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -69,6 +69,8 @@ /*=== Macros ===*/ +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +#define MAX(a,b) ( (a) > (b) ? (a) : (b) ) #define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG)) #define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */ #define DELTANEXTU16(table, pos) table[(U16)(pos)] /* faster */ @@ -610,7 +612,11 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i **************************************/ /* allocation */ LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); } -int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { free(LZ4_streamHCPtr); return 0; } +int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { + if (!LZ4_streamHCPtr) return 0; /* support free on NULL */ + free(LZ4_streamHCPtr); + return 0; +} /* initialization */ @@ -623,6 +629,16 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) LZ4_streamHCPtr->internal_donotuse.searchNum = LZ4HC_getSearchNum(compressionLevel); } +void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) +{ + int const currentCLevel = LZ4_streamHCPtr->internal_donotuse.compressionLevel; + int const minCLevel = currentCLevel < LZ4HC_CLEVEL_OPT_MIN ? 1 : LZ4HC_CLEVEL_OPT_MIN; + int const maxCLevel = currentCLevel < LZ4HC_CLEVEL_OPT_MIN ? LZ4HC_CLEVEL_OPT_MIN-1 : LZ4HC_CLEVEL_MAX; + compressionLevel = MIN(compressionLevel, minCLevel); + compressionLevel = MAX(compressionLevel, maxCLevel); + LZ4_streamHCPtr->internal_donotuse.compressionLevel = compressionLevel; +} + int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) { LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; @@ -767,7 +783,11 @@ void* LZ4_createHC (char* inputBuffer) return hc4; } -int LZ4_freeHC (void* LZ4HC_Data) { FREEMEM(LZ4HC_Data); return 0; } +int LZ4_freeHC (void* LZ4HC_Data) { + if (!LZ4HC_Data) return 0; /* support free on NULL */ + FREEMEM(LZ4HC_Data); + return 0; +} int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int cLevel) { diff --git a/lib/lz4hc.h b/lib/lz4hc.h index b63c825..66d5636 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -161,7 +161,7 @@ typedef struct typedef struct { unsigned int hashTable[LZ4HC_HASHTABLESIZE]; - unsigned short chainTable[LZ4HC_MAXD]; + unsigned short chainTable[LZ4HC_MAXD]; const unsigned char* end; /* next block here to continue on current prefix */ const unsigned char* base; /* All index relative to this position */ const unsigned char* dictBase; /* alternate base for extDict */ @@ -245,25 +245,34 @@ LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStr * or 0 if compression fails. * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src` */ -LZ4LIB_API int LZ4_compress_HC_destSize(void* LZ4HC_Data, +int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* src, char* dst, int* srcSizePtr, int targetDstSize, int compressionLevel); -/*! LZ4_compress_HC_continue_destSize() : +/*! LZ4_compress_HC_continue_destSize() : v1.8.0 (experimental) * Similar as LZ4_compress_HC_continue(), * but will read a variable nb of bytes from `src` * to fit into `targetDstSize` budget. * Result is provided in 2 parts : * @return : the number of bytes written into 'dst' * or 0 if compression fails. - * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src` - * Important : due to limitations, this prototype only works well up to cLevel < LZ4HC_CLEVEL_OPT_MIN - * beyond that level, compression performance will be much reduced due to internal incompatibilities + * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src`. + * Important : due to limitations, this prototype only works well up to cLevel < LZ4HC_CLEVEL_OPT_MIN + * beyond that level, compression performance will be much reduced due to internal incompatibilities */ -LZ4LIB_API int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr, +int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int* srcSizePtr, int targetDstSize); +/*! LZ4_setCompressionLevel() : v1.8.0 (experimental) + * It's possible to change compression level after LZ4_resetStreamHC(), between 2 invocations of LZ4_compress_HC_continue*(), + * but that requires to stay in the same mode (aka 1-10 or 11-12). + * This function ensures this condition. + */ +void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); + + + #endif /* LZ4_HC_SLO_098092834 */ #endif /* LZ4_HC_STATIC_LINKING_ONLY */ diff --git a/tests/frametest.c b/tests/frametest.c index 0dadf9f..d0665c5 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -164,6 +164,9 @@ static unsigned FUZ_highbit(U32 v32) /*-******************************************************* * Tests *********************************************************/ +#define CHECK_V(v,f) v = f; if (LZ4F_isError(v)) goto _output_error +#define CHECK(f) { LZ4F_errorCode_t const CHECK_V(err_ , f); } + int basicTests(U32 seed, double compressibility) { #define COMPRESSIBLE_NOISE_LENGTH (2 MB) @@ -197,24 +200,20 @@ int basicTests(U32 seed, double compressibility) /* Special case : null-content frame */ testSize = 0; DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : "); - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL); - if (LZ4F_isError(cSize)) goto _output_error; + CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL)); DISPLAYLEVEL(3, "null content encoded into a %u bytes frame \n", (unsigned)cSize); DISPLAYLEVEL(3, "LZ4F_createDecompressionContext \n"); - { LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) goto _output_error; } + CHECK ( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) ); DISPLAYLEVEL(3, "LZ4F_getFrameInfo on null-content frame (#157) \n"); { size_t avail_in = cSize; LZ4F_frameInfo_t frame_info; - LZ4F_errorCode_t const errorCode = LZ4F_getFrameInfo(dCtx, &frame_info, compressedBuffer, &avail_in); - if (LZ4F_isError(errorCode)) goto _output_error; + CHECK( LZ4F_getFrameInfo(dCtx, &frame_info, compressedBuffer, &avail_in) ); } DISPLAYLEVEL(3, "LZ4F_freeDecompressionContext \n"); - { LZ4F_errorCode_t const errorCode = LZ4F_freeDecompressionContext(dCtx); - if (LZ4F_isError(errorCode)) goto _output_error; } + CHECK( LZ4F_freeDecompressionContext(dCtx) ); dCtx = NULL; /* test one-pass frame compression */ @@ -224,26 +223,22 @@ int basicTests(U32 seed, double compressibility) { LZ4F_preferences_t fastCompressPrefs; memset(&fastCompressPrefs, 0, sizeof(fastCompressPrefs)); fastCompressPrefs.compressionLevel = -3; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, &fastCompressPrefs); - if (LZ4F_isError(cSize)) goto _output_error; + CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, &fastCompressPrefs)); DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize); } DISPLAYLEVEL(3, "LZ4F_compressFrame, using default preferences : "); - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL); - if (LZ4F_isError(cSize)) goto _output_error; + CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL)); DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize); DISPLAYLEVEL(3, "Decompression test : \n"); { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; size_t compressedBufferSize = cSize; - LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) goto _output_error; + CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) ); DISPLAYLEVEL(3, "Single Pass decompression : "); - { size_t const decompressError = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL); - if (LZ4F_isError(decompressError)) goto _output_error; } + CHECK( LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL) ); { U64 const crcDest = XXH64(decodedBuffer, decodedBufferSize, 1); if (crcDest != crcOrig) goto _output_error; } DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedBufferSize); @@ -257,8 +252,7 @@ int basicTests(U32 seed, double compressibility) BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; size_t decResult, oSize = COMPRESSIBLE_NOISE_LENGTH; DISPLAYLEVEL(3, "Missing last %u bytes : ", (U32)missingBytes); - decResult = LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL); - if (LZ4F_isError(decResult)) goto _output_error; + CHECK_V(decResult, LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL)); if (decResult != missingBytes) { DISPLAY("%u bytes missing != %u bytes requested \n", (U32)missingBytes, (U32)decResult); goto _output_error; @@ -282,9 +276,9 @@ int basicTests(U32 seed, double compressibility) const BYTE* ip = (BYTE*)compressedBuffer; DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : "); - errorCode = LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; - DISPLAYLEVEL(3, " %u \n", (unsigned)errorCode); + CHECK( LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL) ); + //DISPLAYLEVEL(3, " %u \n", (unsigned)errorCode); + DISPLAYLEVEL(3, " OK \n"); DISPLAYLEVEL(3, "LZ4F_getFrameInfo on zero-size input : "); { size_t nullSize = 0; @@ -309,8 +303,7 @@ int basicTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "LZ4F_getFrameInfo on enough input : "); iSize = 15 - iSize; - errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize); - if (LZ4F_isError(errorCode)) goto _output_error; + CHECK( LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize) ); DISPLAYLEVEL(3, " correctly decoded \n"); ip += iSize; } @@ -342,8 +335,7 @@ int basicTests(U32 seed, double compressibility) while (ip < iend) { size_t oSize = oend-op; size_t iSize = 1; - errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; + CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); op += oSize; ip += iSize; } @@ -356,21 +348,18 @@ int basicTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "Using 64 KB block : "); prefs.frameInfo.blockSizeID = LZ4F_max64KB; prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; + CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs)); DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "without checksum : "); prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; + CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs)); DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "Using 256 KB block : "); prefs.frameInfo.blockSizeID = LZ4F_max256KB; prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; + CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs)); DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "Decompression test : \n"); @@ -388,8 +377,7 @@ int basicTests(U32 seed, double compressibility) size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; size_t oSize = oend-op; if (iSize > (size_t)(iend-ip)) iSize = iend-ip; - { size_t const decompressError = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); - if (LZ4F_isError(decompressError)) goto _output_error; } + CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); op += oSize; ip += iSize; } @@ -399,28 +387,24 @@ int basicTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize); } - { LZ4F_errorCode_t const freeError = LZ4F_freeDecompressionContext(dCtx); - if (LZ4F_isError(freeError)) goto _output_error; } + CHECK( LZ4F_freeDecompressionContext(dCtx) ); dCtx = NULL; } DISPLAYLEVEL(3, "without checksum : "); prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; + CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) ); DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "Using 1 MB block : "); prefs.frameInfo.blockSizeID = LZ4F_max1MB; prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; + CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) ); DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "without checksum : "); prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; + CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) ); DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "Using 4 MB block : "); @@ -428,8 +412,7 @@ int basicTests(U32 seed, double compressibility) prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; { size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs); DISPLAYLEVEL(4, "dstCapacity = %u ; ", (U32)dstCapacity) - cSize = LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; + CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs) ); DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize); } @@ -437,61 +420,213 @@ int basicTests(U32 seed, double compressibility) prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; { size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs); DISPLAYLEVEL(4, "dstCapacity = %u ; ", (U32)dstCapacity) - cSize = LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; + CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs) ); DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize); } - { size_t errorCode; + /* frame content size tests */ + { size_t cErr; BYTE* const ostart = (BYTE*)compressedBuffer; BYTE* op = ostart; - errorCode = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) goto _output_error; + CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) ); DISPLAYLEVEL(3, "compress without frameSize : "); memset(&(prefs.frameInfo), 0, sizeof(prefs.frameInfo)); - errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs); - if (LZ4F_isError(errorCode)) goto _output_error; - op += errorCode; - errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; - op += errorCode; - errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; + CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs)); + op += cErr; + CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL)); + op += cErr; + CHECK( LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL) ); DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart)); DISPLAYLEVEL(3, "compress with frameSize : "); prefs.frameInfo.contentSize = testSize; op = ostart; - errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs); - if (LZ4F_isError(errorCode)) goto _output_error; - op += errorCode; - errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; - op += errorCode; - errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; + CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs)); + op += cErr; + CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL)); + op += cErr; + CHECK( LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL) ); DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart)); DISPLAYLEVEL(3, "compress with wrong frameSize : "); prefs.frameInfo.contentSize = testSize+1; op = ostart; - errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs); - if (LZ4F_isError(errorCode)) goto _output_error; - op += errorCode; - errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; - op += errorCode; - errorCode = LZ4F_compressEnd(cctx, op, testSize, NULL); - if (LZ4F_isError(errorCode)) { DISPLAYLEVEL(3, "Error correctly detected : %s \n", LZ4F_getErrorName(errorCode)); } - else - goto _output_error; - - errorCode = LZ4F_freeCompressionContext(cctx); - if (LZ4F_isError(errorCode)) goto _output_error; + CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs)); + op += cErr; + CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL)); + op += cErr; + cErr = LZ4F_compressEnd(cctx, op, testSize, NULL); + if (!LZ4F_isError(cErr)) goto _output_error; + DISPLAYLEVEL(3, "Error correctly detected : %s \n", LZ4F_getErrorName(cErr)); + + CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL; } + /* dictID tests */ + { size_t cErr; + U32 const dictID = 0x99; + CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) ); + + DISPLAYLEVEL(3, "insert a dictID : "); + memset(&prefs.frameInfo, 0, sizeof(prefs.frameInfo)); + prefs.frameInfo.dictID = dictID; + CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs)); + DISPLAYLEVEL(3, "created frame header of size %i bytes \n", (int)cErr); + + DISPLAYLEVEL(3, "read a dictID : "); + CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) ); + memset(&prefs.frameInfo, 0, sizeof(prefs.frameInfo)); + CHECK( LZ4F_getFrameInfo(dCtx, &prefs.frameInfo, compressedBuffer, &cErr) ); + if (prefs.frameInfo.dictID != dictID) goto _output_error; + DISPLAYLEVEL(3, "%u \n", (U32)prefs.frameInfo.dictID); + + CHECK( LZ4F_freeDecompressionContext(dCtx) ); dCtx = NULL; + CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL; + } + + + /* Dictionary compression test */ + { size_t const dictSize = 63 KB; + size_t const dstCapacity = LZ4F_compressFrameBound(dictSize, NULL); + size_t cSizeNoDict, cSizeWithDict; + LZ4F_CDict* const cdict = LZ4F_createCDict(CNBuffer, dictSize); + if (cdict == NULL) goto _output_error; + DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with NULL dict : "); + CHECK_V(cSizeNoDict, + LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity, + CNBuffer, dictSize, + NULL, NULL) ); + DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeNoDict); + + DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict : "); + CHECK_V(cSizeWithDict, + LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity, + CNBuffer, dictSize, + cdict, NULL) ); + DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n", + (unsigned)dictSize, (unsigned)cSizeWithDict); + if (cSizeWithDict >= cSizeNoDict) goto _output_error; /* must be more efficient */ + crcOrig = XXH64(CNBuffer, dictSize, 0); + + DISPLAYLEVEL(3, "LZ4F_decompress_usingDict : "); + { LZ4F_dctx* dctx; + size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH; + size_t compressedSize = cSizeWithDict; + CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) ); + CHECK( LZ4F_decompress_usingDict(dctx, + decodedBuffer, &decodedSize, + compressedBuffer, &compressedSize, + CNBuffer, dictSize, + NULL) ); + if (compressedSize != cSizeWithDict) goto _output_error; + if (decodedSize != dictSize) goto _output_error; + { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0); + if (crcDest != crcOrig) goto _output_error; } + DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize); + CHECK( LZ4F_freeDecompressionContext(dctx) ); + } + + DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict, negative level : "); + { size_t cSizeLevelMax; + LZ4F_preferences_t cParams; + memset(&cParams, 0, sizeof(cParams)); + cParams.compressionLevel = -3; + CHECK_V(cSizeLevelMax, + LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity, + CNBuffer, dictSize, + cdict, &cParams) ); + DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax); + } + + DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict, level max : "); + { size_t cSizeLevelMax; + LZ4F_preferences_t cParams; + memset(&cParams, 0, sizeof(cParams)); + cParams.compressionLevel = LZ4F_compressionLevel_max(); + CHECK_V(cSizeLevelMax, + LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity, + CNBuffer, dictSize, + cdict, &cParams) ); + DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax); + } + + DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, multiple linked blocks : "); + { size_t cSizeContiguous; + size_t const inSize = dictSize * 3; + size_t const outCapacity = LZ4F_compressFrameBound(inSize, NULL); + LZ4F_preferences_t cParams; + memset(&cParams, 0, sizeof(cParams)); + cParams.frameInfo.blockMode = LZ4F_blockLinked; + cParams.frameInfo.blockSizeID = LZ4F_max64KB; + CHECK_V(cSizeContiguous, + LZ4F_compressFrame_usingCDict(compressedBuffer, outCapacity, + CNBuffer, inSize, + cdict, &cParams) ); + DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n", + (unsigned)inSize, (unsigned)cSizeContiguous); + + DISPLAYLEVEL(3, "LZ4F_decompress_usingDict on multiple linked blocks : "); + { LZ4F_dctx* dctx; + size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH; + size_t compressedSize = cSizeContiguous; + CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) ); + CHECK( LZ4F_decompress_usingDict(dctx, + decodedBuffer, &decodedSize, + compressedBuffer, &compressedSize, + CNBuffer, dictSize, + NULL) ); + if (compressedSize != cSizeContiguous) goto _output_error; + if (decodedSize != inSize) goto _output_error; + crcOrig = XXH64(CNBuffer, inSize, 0); + { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0); + if (crcDest != crcOrig) goto _output_error; } + DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize); + CHECK( LZ4F_freeDecompressionContext(dctx) ); + } + } + + + DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, multiple independent blocks : "); + { size_t cSizeIndep; + size_t const inSize = dictSize * 3; + size_t const outCapacity = LZ4F_compressFrameBound(inSize, NULL); + LZ4F_preferences_t cParams; + memset(&cParams, 0, sizeof(cParams)); + cParams.frameInfo.blockMode = LZ4F_blockIndependent; + cParams.frameInfo.blockSizeID = LZ4F_max64KB; + CHECK_V(cSizeIndep, + LZ4F_compressFrame_usingCDict(compressedBuffer, outCapacity, + CNBuffer, inSize, + cdict, &cParams) ); + DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n", + (unsigned)inSize, (unsigned)cSizeIndep); + + DISPLAYLEVEL(3, "LZ4F_decompress_usingDict on multiple independent blocks : "); + { LZ4F_dctx* dctx; + size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH; + size_t compressedSize = cSizeIndep; + CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) ); + CHECK( LZ4F_decompress_usingDict(dctx, + decodedBuffer, &decodedSize, + compressedBuffer, &compressedSize, + CNBuffer, dictSize, + NULL) ); + if (compressedSize != cSizeIndep) goto _output_error; + if (decodedSize != inSize) goto _output_error; + crcOrig = XXH64(CNBuffer, inSize, 0); + { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0); + if (crcDest != crcOrig) goto _output_error; } + DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize); + CHECK( LZ4F_freeDecompressionContext(dctx) ); + } + } + + LZ4F_freeCDict(cdict); + } + + DISPLAYLEVEL(3, "Skippable frame test : \n"); { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; unsigned maxBits = FUZ_highbit((U32)decodedBufferSize); @@ -500,8 +635,7 @@ int basicTests(U32 seed, double compressibility) BYTE* ip = (BYTE*)compressedBuffer; BYTE* iend = (BYTE*)compressedBuffer + cSize + 8; - LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) goto _output_error; + CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) ); /* generate skippable frame */ FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START); @@ -513,8 +647,7 @@ int basicTests(U32 seed, double compressibility) size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; size_t oSize = oend-op; if (iSize > (size_t)(iend-ip)) iSize = iend-ip; - errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; + CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); op += oSize; ip += iSize; } @@ -533,8 +666,7 @@ int basicTests(U32 seed, double compressibility) size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; size_t oSize = oend-op; if (iSize > (size_t)(iend-ip)) iSize = iend-ip; - errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; + CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); op += oSize; ip += iSize; } @@ -550,8 +682,7 @@ int basicTests(U32 seed, double compressibility) size_t iSize = 10; size_t oSize = 10; if (iSize > (size_t)(iend-ip)) iSize = iend-ip; - errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; + CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); op += oSize; ip += iSize; } @@ -603,6 +734,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi size_t result; clock_t const startClock = clock(); clock_t const clockDuration = duration_s * CLOCKS_PER_SEC; +# undef CHECK # define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } |