summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorYann Collet <cyan@fb.com>2017-08-15 23:57:00 (GMT)
committerYann Collet <cyan@fb.com>2017-08-15 23:57:00 (GMT)
commita1f3a27e9b3caf42d31100f96bbb88866e08a517 (patch)
tree014922fceee22f63d46a0d04328234c3711f4799
parent01cdbfb5feda4adee65cc3da04b426c0c22368a3 (diff)
parentaf9d72b7f6809ec972787c446d464a8f9be443ba (diff)
downloadlz4-a1f3a27e9b3caf42d31100f96bbb88866e08a517.zip
lz4-a1f3a27e9b3caf42d31100f96bbb88866e08a517.tar.gz
lz4-a1f3a27e9b3caf42d31100f96bbb88866e08a517.tar.bz2
Merge branch 'dev' into installVars and fixed conflicts
-rw-r--r--NEWS1
-rw-r--r--doc/lz4_Frame_format.md3
-rw-r--r--doc/lz4_manual.html34
-rw-r--r--doc/lz4frame_manual.html141
-rw-r--r--examples/frameCompress.c537
-rw-r--r--lib/lz4.c7
-rw-r--r--lib/lz4.h38
-rw-r--r--lib/lz4frame.c1063
-rw-r--r--lib/lz4frame.h132
-rw-r--r--lib/lz4frame_static.h56
-rw-r--r--lib/lz4hc.c24
-rw-r--r--lib/lz4hc.h23
-rw-r--r--programs/lz4cli.c2
-rw-r--r--programs/lz4io.c11
-rw-r--r--tests/Makefile1
-rw-r--r--tests/frametest.c330
16 files changed, 1431 insertions, 972 deletions
diff --git a/NEWS b/NEWS
index d10c48f..63d53a6 100644
--- a/NEWS
+++ b/NEWS
@@ -4,6 +4,7 @@ cli : added GNU separator -- specifying that all following arguments are files
API : added LZ4_compress_HC_destSize(), by Oleg (@remittor)
API : added LZ4F_resetDecompressionContext()
API : lz4frame : negative compression levels trigger fast acceleration, request by Lawrence Chan
+API : lz4frame : can control block checksum and dictionary ID
API : fix : expose obsolete decoding functions, reported by Chen Yufei
build : fix : static lib installation, by Ido Rosen
build : dragonFlyBSD, OpenBSD, NetBSD supported
diff --git a/doc/lz4_Frame_format.md b/doc/lz4_Frame_format.md
index bae2b06..77454b2 100644
--- a/doc/lz4_Frame_format.md
+++ b/doc/lz4_Frame_format.md
@@ -237,8 +237,7 @@ __Header Checksum__
One-byte checksum of combined descriptor fields, including optional ones.
The value is the second byte of `xxh32()` : ` (xxh32()>>8) & 0xFF `
-using zero as a seed,
-and the full Frame Descriptor as an input
+using zero as a seed, and the full Frame Descriptor as an input
(including optional fields when they are present).
A wrong checksum indicates an error in the descriptor.
Header checksum is informational and can be skipped.
diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index fecd8cd..9ab1984 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -175,8 +175,8 @@ int LZ4_freeStream (LZ4_stream_t* streamPtr);
Important : Previous data blocks are assumed to remain present and unmodified !
'dst' buffer must be already allocated.
If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
- If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero.
- After an error, the stream status is invalid, and it can only be reset or freed.
+ If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function @return==0.
+ After an error, the stream status is invalid, it can only be reset or freed.
</p></pre><BR>
@@ -205,20 +205,20 @@ int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
<pre><b>int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize);
-</b><p> These decoding functions allow decompression of multiple blocks in "streaming" mode.
- Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
- In the case of a ring buffers, decoding buffer must be either :
- - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
- In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
- - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
- maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
- In which case, encoding and decoding buffers do not need to be synchronized,
- and encoding ring buffer can have any size, including small ones ( < 64 KB).
- - _At least_ 64 KB + 8 bytes + maxBlockSize.
- In which case, encoding and decoding buffers do not need to be synchronized,
- and encoding ring buffer can have any size, including larger than decoding buffer.
- Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
- and indicate where it is saved using LZ4_setStreamDecode()
+</b><p> These decoding functions allow decompression of multiple blocks in "streaming" mode.
+ Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
+ In the case of a ring buffers, decoding buffer must be either :
+ - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
+ In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
+ - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
+ maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
+ In which case, encoding and decoding buffers do not need to be synchronized,
+ and encoding ring buffer can have any size, including small ones ( < 64 KB).
+ - _At least_ 64 KB + 8 bytes + maxBlockSize.
+ In which case, encoding and decoding buffers do not need to be synchronized,
+ and encoding ring buffer can have any size, including larger than decoding buffer.
+ Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
+ and indicate where it is saved using LZ4_setStreamDecode()
</p></pre><BR>
<pre><b>int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize);
@@ -277,7 +277,7 @@ union LZ4_stream_u {
init this structure before first use.
note : only use in association with static linking !
this definition is not API/ABI safe,
- and may change in a future version !
+ it may change in a future version !
</p></pre><BR>
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index b82dfe5..86aaf3d 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -63,52 +63,58 @@
} LZ4F_contentChecksum_t;
</b></pre><BR>
<pre><b>typedef enum {
+ LZ4F_noBlockChecksum=0,
+ LZ4F_blockChecksumEnabled
+} LZ4F_blockChecksum_t;
+</b></pre><BR>
+<pre><b>typedef enum {
LZ4F_frame=0,
LZ4F_skippableFrame
LZ4F_OBSOLETE_ENUM(skippableFrame)
} LZ4F_frameType_t;
</b></pre><BR>
<pre><b>typedef struct {
- LZ4F_blockSizeID_t blockSizeID; </b>/* max64KB, max256KB, max1MB, max4MB ; 0 == default */<b>
- LZ4F_blockMode_t blockMode; </b>/* blockLinked, blockIndependent ; 0 == default */<b>
- LZ4F_contentChecksum_t contentChecksumFlag; </b>/* noContentChecksum, contentChecksumEnabled ; 0 == default */<b>
- LZ4F_frameType_t frameType; </b>/* LZ4F_frame, skippableFrame ; 0 == default */<b>
- unsigned long long contentSize; </b>/* Size of uncompressed (original) content ; 0 == unknown */<b>
- unsigned reserved[2]; </b>/* must be zero for forward compatibility */<b>
+ LZ4F_blockSizeID_t blockSizeID; </b>/* max64KB, max256KB, max1MB, max4MB ; 0 == default */<b>
+ LZ4F_blockMode_t blockMode; </b>/* LZ4F_blockLinked, LZ4F_blockIndependent ; 0 == default */<b>
+ LZ4F_contentChecksum_t contentChecksumFlag; </b>/* if enabled, frame is terminated with a 32-bits checksum of decompressed data ; 0 == disabled (default) */<b>
+ LZ4F_frameType_t frameType; </b>/* read-only field : LZ4F_frame or LZ4F_skippableFrame */<b>
+ unsigned long long contentSize; </b>/* Size of uncompressed content ; 0 == unknown */<b>
+ unsigned dictID; </b>/* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */<b>
+ LZ4F_blockChecksum_t blockChecksumFlag; </b>/* if enabled, each block is followed by a checksum of block's compressed data ; 0 == disabled (default) */<b>
} LZ4F_frameInfo_t;
-</b><p> makes it possible to supply detailed frame parameters to the stream interface.
- It's not required to set all fields, as long as the structure was initially memset() to zero.
- All reserved fields must be set to zero.
+</b><p> makes it possible to set or read frame parameters.
+ It's not required to set all fields, as long as the structure was initially memset() to zero.
+ For all fields, 0 sets it to default value
</p></pre><BR>
<pre><b>typedef struct {
LZ4F_frameInfo_t frameInfo;
int compressionLevel; </b>/* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */<b>
- unsigned autoFlush; </b>/* 1 == always flush (reduce usage of tmp buffer) */<b>
+ unsigned autoFlush; </b>/* 1 == always flush, to reduce usage of internal buffers */<b>
unsigned reserved[4]; </b>/* must be zero for forward compatibility */<b>
} LZ4F_preferences_t;
-</b><p> makes it possible to supply detailed compression parameters to the stream interface.
- It's not required to set all fields, as long as the structure was initially memset() to zero.
- All reserved fields must be set to zero.
+</b><p> makes it possible to supply detailed compression parameters to the stream interface.
+ It's not required to set all fields, as long as the structure was initially memset() to zero.
+ All reserved fields must be set to zero.
</p></pre><BR>
<a name="Chapter5"></a><h2>Simple compression function</h2><pre></pre>
<pre><b>size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
-</b><p> Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
- Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
-
+</b><p> Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
+ Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
+
</p></pre><BR>
-<pre><b>size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
-</b><p> Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.1
- An important rule is that dstBuffer MUST be large enough (dstCapacity) to store the result in worst case situation.
- This value is supplied by LZ4F_compressFrameBound().
- If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode).
- The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
+<pre><b>size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
+ const void* srcBuffer, size_t srcSize,
+ const LZ4F_preferences_t* preferencesPtr);
+</b><p> Compress an entire srcBuffer into a valid LZ4 frame.
+ dstCapacity MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
+ The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
@return : number of bytes written into dstBuffer.
or an error code if it fails (can be tested using LZ4F_isError())
-
+
</p></pre><BR>
<a name="Chapter6"></a><h2>Advanced compression functions</h2><pre></pre>
@@ -128,18 +134,20 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
The function will provide a pointer to a fully allocated LZ4F_cctx object.
If @return != zero, there was an error during context creation.
Object can release its memory using LZ4F_freeCompressionContext();
-
+
</p></pre><BR>
<a name="Chapter8"></a><h2>Compression</h2><pre></pre>
-<pre><b>size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* prefsPtr);
-</b><p> will write the frame header into dstBuffer.
- dstCapacity must be large enough to store the header. Maximum header size is LZ4F_HEADER_SIZE_MAX bytes.
+<pre><b>size_t LZ4F_compressBegin(LZ4F_cctx* cctx,
+ void* dstBuffer, size_t dstCapacity,
+ const LZ4F_preferences_t* prefsPtr);
+</b><p> will write the frame header into dstBuffer.
+ dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
`prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default.
@return : number of bytes written into dstBuffer for the header
or an error code (which can be tested using LZ4F_isError())
-
+
</p></pre><BR>
<pre><b>size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr);
@@ -147,7 +155,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario.
Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
-
+
</p></pre><BR>
<pre><b>size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr);
@@ -159,7 +167,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
`cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
@return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
or an error code if it fails (which can be tested using LZ4F_isError())
-
+
</p></pre><BR>
<pre><b>size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
@@ -169,25 +177,25 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
`cOptPtr` is optional : it's possible to provide NULL, all options will be set to default.
@return : number of bytes written into dstBuffer (it can be zero, which means there was no data stored within cctx)
or an error code if it fails (which can be tested using LZ4F_isError())
-
+
</p></pre><BR>
<pre><b>size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
-</b><p> To properly finish an LZ4 frame, invoke LZ4F_compressEnd().
- It will flush whatever data remained within `cctx` (like LZ4_flush())
- and properly finalize the frame, with an endMark and a checksum.
+</b><p> To properly finish an LZ4 frame, invoke LZ4F_compressEnd().
+ It will flush whatever data remained within `cctx` (like LZ4_flush())
+ and properly finalize the frame, with an endMark and a checksum.
`cOptPtr` is optional : NULL can be provided, in which case all options will be set to default.
@return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled)
or an error code if it fails (which can be tested using LZ4F_isError())
- A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.
-
+ A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.
+
</p></pre><BR>
<a name="Chapter9"></a><h2>Decompression functions</h2><pre></pre>
<pre><b>typedef struct {
unsigned stableDst; </b>/* pledge that at least 64KB+64Bytes of previously decompressed data remain unmodifed where it was decoded. This optimization skips storage operations in tmp buffers */<b>
- unsigned reserved[3];
+ unsigned reserved[3]; </b>/* must be set to zero for forward compatibility */<b>
} LZ4F_decompressOptions_t;
</b></pre><BR>
<pre><b>LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version);
@@ -199,7 +207,7 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
dctx memory can be released using LZ4F_freeDecompressionContext();
The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
That is, it should be == 0 if decompression has been completed fully and correctly.
-
+
</p></pre><BR>
<a name="Chapter10"></a><h2>Streaming decompression functions</h2><pre></pre>
@@ -207,10 +215,10 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
<pre><b>size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
LZ4F_frameInfo_t* frameInfoPtr,
const void* srcBuffer, size_t* srcSizePtr);
-</b><p> This function extracts frame parameters (such as max blockSize, frame checksum, etc.).
- Its usage is optional.
- Extracted information can typically be useful for allocation purposes.
- This function works in 2 situations :
+</b><p> This function extracts frame parameters (max blockSize, dictID, etc.).
+ Its usage is optional.
+ Extracted information is typically useful for allocation and dictionary.
+ This function works in 2 situations :
- At the beginning of a new frame, in which case
it will decode information from `srcBuffer`, starting the decoding process.
Input size must be large enough to successfully decode the entire frame header.
@@ -218,50 +226,51 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
It's allowed to provide more input data than this minimum.
- After decoding has been started.
In which case, no input is read, frame parameters are extracted from dctx.
- - If decoding has barely started, but not yet extracted information from header, LZ4F_getFrameInfo() will fail.
- The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
- Decompression must resume from (srcBuffer + *srcSizePtr).
+ - If decoding has barely started, but not yet extracted information from header,
+ LZ4F_getFrameInfo() will fail.
+ The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
+ Decompression must resume from (srcBuffer + *srcSizePtr).
@return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
- or an error code which can be tested using LZ4F_isError()
- note 1 : in case of error, dctx is not modified. Decoding operation can resume safely.
- note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
-
+ or an error code which can be tested using LZ4F_isError().
+ note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely.
+ note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
+
</p></pre><BR>
<pre><b>size_t LZ4F_decompress(LZ4F_dctx* dctx,
void* dstBuffer, size_t* dstSizePtr,
const void* srcBuffer, size_t* srcSizePtr,
const LZ4F_decompressOptions_t* dOptPtr);
-</b><p> Call this function repetitively to regenerate compressed data from `srcBuffer`.
- The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
+</b><p> Call this function repetitively to regenerate compressed data from `srcBuffer`.
+ The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
- The number of bytes regenerated into dstBuffer is provided within *dstSizePtr (necessarily <= original value).
+ The number of bytes regenerated into dstBuffer is provided within *dstSizePtr (necessarily <= original value).
- The number of bytes consumed from srcBuffer is provided within *srcSizePtr (necessarily <= original value).
- Number of bytes consumed can be < number of bytes provided.
- It typically happens when dstBuffer is not large enough to contain all decoded data.
- Unconsumed source data must be presented again in subsequent invocations.
+ The number of bytes consumed from srcBuffer is provided within *srcSizePtr (necessarily <= original value).
+ Number of bytes consumed can be < number of bytes provided.
+ It typically happens when dstBuffer is not large enough to contain all decoded data.
+ Unconsumed source data must be presented again in subsequent invocations.
`dstBuffer` content is expected to be flushed between each invocation, as its content will be overwritten.
`dstBuffer` itself can be changed at will between each consecutive function invocation.
- @return is an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
- Schematically, it's the size of the current (or remaining) compressed block + header of next block.
- Respecting the hint provides some small speed benefit, because it skips intermediate buffers.
- This is just a hint though, it's always possible to provide any srcSize.
- When a frame is fully decoded, @return will be 0 (no more data expected).
- If decompression failed, @return is an error code, which can be tested using LZ4F_isError().
+ @return : an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
+ Schematically, it's the size of the current (or remaining) compressed block + header of next block.
+ Respecting the hint provides some small speed benefit, because it skips intermediate buffers.
+ This is just a hint though, it's always possible to provide any srcSize.
+ When a frame is fully decoded, @return will be 0 (no more data expected).
+ If decompression failed, @return is an error code, which can be tested using LZ4F_isError().
+
+ After a frame is fully decoded, dctx can be used again to decompress another frame.
+ After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state.
- After a frame is fully decoded, dctx can be used again to decompress another frame.
- After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state.
-
</p></pre><BR>
<pre><b>void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx); </b>/* always successful */<b>
</b><p> In case of an error, the context is left in "undefined" state.
In which case, it's necessary to reset it, before re-using it.
This method can also be used to abruptly stop an unfinished decompression,
- and start a new one using the same context.
+ and start a new one using the same context.
</p></pre><BR>
</html>
diff --git a/examples/frameCompress.c b/examples/frameCompress.c
index 75f1576..8712725 100644
--- a/examples/frameCompress.c
+++ b/examples/frameCompress.c
@@ -13,299 +13,300 @@
#define LZ4_FOOTER_SIZE 4
static const LZ4F_preferences_t lz4_preferences = {
- { LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame, 0, { 0, 0 } },
- 0, /* compression level */
- 0, /* autoflush */
- { 0, 0, 0, 0 }, /* reserved, must be set to 0 */
+ { LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame,
+ 0 /* content size unknown */, 0 /* no dictID */ , LZ4F_noBlockChecksum },
+ 0, /* compression level */
+ 0, /* autoflush */
+ { 0, 0, 0, 0 }, /* reserved, must be set to 0 */
};
static size_t compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) {
- LZ4F_errorCode_t r;
- LZ4F_compressionContext_t ctx;
- char *src, *buf = NULL;
- size_t size, n, k, count_in = 0, count_out, offset = 0, frame_size;
-
- r = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
- if (LZ4F_isError(r)) {
- printf("Failed to create context: error %zu\n", r);
- return 1;
- }
- r = 1;
-
- src = malloc(BUF_SIZE);
- if (!src) {
- printf("Not enough memory\n");
- goto cleanup;
- }
-
- frame_size = LZ4F_compressBound(BUF_SIZE, &lz4_preferences);
- size = frame_size + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE;
- buf = malloc(size);
- if (!buf) {
- printf("Not enough memory\n");
- goto cleanup;
- }
-
- n = offset = count_out = LZ4F_compressBegin(ctx, buf, size, &lz4_preferences);
- if (LZ4F_isError(n)) {
- printf("Failed to start compression: error %zu\n", n);
- goto cleanup;
- }
-
- printf("Buffer size is %zu bytes, header size %zu bytes\n", size, n);
-
- for (;;) {
- k = fread(src, 1, BUF_SIZE, in);
- if (k == 0)
- break;
- count_in += k;
-
- n = LZ4F_compressUpdate(ctx, buf + offset, size - offset, src, k, NULL);
- if (LZ4F_isError(n)) {
- printf("Compression failed: error %zu\n", n);
- goto cleanup;
- }
-
- offset += n;
- count_out += n;
- if (size - offset < frame_size + LZ4_FOOTER_SIZE) {
- printf("Writing %zu bytes\n", offset);
-
- k = fwrite(buf, 1, offset, out);
- if (k < offset) {
- if (ferror(out))
- printf("Write failed\n");
- else
- printf("Short write\n");
- goto cleanup;
- }
-
- offset = 0;
- }
- }
-
- n = LZ4F_compressEnd(ctx, buf + offset, size - offset, NULL);
- if (LZ4F_isError(n)) {
- printf("Failed to end compression: error %zu\n", n);
- goto cleanup;
- }
-
- offset += n;
- count_out += n;
- printf("Writing %zu bytes\n", offset);
-
- k = fwrite(buf, 1, offset, out);
- if (k < offset) {
- if (ferror(out))
- printf("Write failed\n");
- else
- printf("Short write\n");
- goto cleanup;
- }
-
- *size_in = count_in;
- *size_out = count_out;
- r = 0;
+ LZ4F_errorCode_t r;
+ LZ4F_compressionContext_t ctx;
+ char *src, *buf = NULL;
+ size_t size, n, k, count_in = 0, count_out, offset = 0, frame_size;
+
+ r = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
+ if (LZ4F_isError(r)) {
+ printf("Failed to create context: error %zu\n", r);
+ return 1;
+ }
+ r = 1; /* function result; 1 == error, by default (early exit) */
+
+ src = malloc(BUF_SIZE);
+ if (!src) {
+ printf("Not enough memory\n");
+ goto cleanup;
+ }
+
+ frame_size = LZ4F_compressBound(BUF_SIZE, &lz4_preferences);
+ size = frame_size + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE;
+ buf = malloc(size);
+ if (!buf) {
+ printf("Not enough memory\n");
+ goto cleanup;
+ }
+
+ n = offset = count_out = LZ4F_compressBegin(ctx, buf, size, &lz4_preferences);
+ if (LZ4F_isError(n)) {
+ printf("Failed to start compression: error %zu\n", n);
+ goto cleanup;
+ }
+
+ printf("Buffer size is %zu bytes, header size %zu bytes\n", size, n);
+
+ for (;;) {
+ k = fread(src, 1, BUF_SIZE, in);
+ if (k == 0)
+ break;
+ count_in += k;
+
+ n = LZ4F_compressUpdate(ctx, buf + offset, size - offset, src, k, NULL);
+ if (LZ4F_isError(n)) {
+ printf("Compression failed: error %zu\n", n);
+ goto cleanup;
+ }
+
+ offset += n;
+ count_out += n;
+ if (size - offset < frame_size + LZ4_FOOTER_SIZE) {
+ printf("Writing %zu bytes\n", offset);
+
+ k = fwrite(buf, 1, offset, out);
+ if (k < offset) {
+ if (ferror(out))
+ printf("Write failed\n");
+ else
+ printf("Short write\n");
+ goto cleanup;
+ }
+
+ offset = 0;
+ }
+ }
+
+ n = LZ4F_compressEnd(ctx, buf + offset, size - offset, NULL);
+ if (LZ4F_isError(n)) {
+ printf("Failed to end compression: error %zu\n", n);
+ goto cleanup;
+ }
+
+ offset += n;
+ count_out += n;
+ printf("Writing %zu bytes\n", offset);
+
+ k = fwrite(buf, 1, offset, out);
+ if (k < offset) {
+ if (ferror(out))
+ printf("Write failed\n");
+ else
+ printf("Short write\n");
+ goto cleanup;
+ }
+
+ *size_in = count_in;
+ *size_out = count_out;
+ r = 0;
cleanup:
- if (ctx)
- LZ4F_freeCompressionContext(ctx);
- free(src);
- free(buf);
- return r;
+ if (ctx)
+ LZ4F_freeCompressionContext(ctx);
+ free(src);
+ free(buf);
+ return r;
}
static size_t get_block_size(const LZ4F_frameInfo_t* info) {
- switch (info->blockSizeID) {
+ switch (info->blockSizeID) {
case LZ4F_default:
- case LZ4F_max64KB: return 1 << 16;
- case LZ4F_max256KB: return 1 << 18;
- case LZ4F_max1MB: return 1 << 20;
- case LZ4F_max4MB: return 1 << 22;
- default:
- printf("Impossible unless more block sizes are allowed\n");
- exit(1);
- }
+ case LZ4F_max64KB: return 1 << 16;
+ case LZ4F_max256KB: return 1 << 18;
+ case LZ4F_max1MB: return 1 << 20;
+ case LZ4F_max4MB: return 1 << 22;
+ default:
+ printf("Impossible unless more block sizes are allowed\n");
+ exit(1);
+ }
}
static size_t decompress_file(FILE *in, FILE *out) {
- void* const src = malloc(BUF_SIZE);
- void* dst = NULL;
- size_t dstCapacity = 0;
- LZ4F_dctx *dctx = NULL;
- size_t ret;
+ void* const src = malloc(BUF_SIZE);
+ void* dst = NULL;
+ size_t dstCapacity = 0;
+ LZ4F_dctx *dctx = NULL;
+ size_t ret;
- /* Initialization */
+ /* Initialization */
if (!src) { perror("decompress_file(src)"); goto cleanup; }
- ret = LZ4F_createDecompressionContext(&dctx, 100);
- if (LZ4F_isError(ret)) {
- printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(ret));
- goto cleanup;
- }
-
- /* Decompression */
- ret = 1;
- while (ret != 0) {
- /* Load more input */
- size_t srcSize = fread(src, 1, BUF_SIZE, in);
- void* srcPtr = src;
- void* srcEnd = srcPtr + srcSize;
- if (srcSize == 0 || ferror(in)) {
- printf("Decompress: not enough input or error reading file\n");
- goto cleanup;
- }
- /* Allocate destination buffer if it isn't already */
- if (!dst) {
- LZ4F_frameInfo_t info;
- ret = LZ4F_getFrameInfo(dctx, &info, src, &srcSize);
- if (LZ4F_isError(ret)) {
- printf("LZ4F_getFrameInfo error: %s\n", LZ4F_getErrorName(ret));
- goto cleanup;
- }
- /* Allocating enough space for an entire block isn't necessary for
- * correctness, but it allows some memcpy's to be elided.
- */
- dstCapacity = get_block_size(&info);
- dst = malloc(dstCapacity);
+ ret = LZ4F_createDecompressionContext(&dctx, 100);
+ if (LZ4F_isError(ret)) {
+ printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(ret));
+ goto cleanup;
+ }
+
+ /* Decompression */
+ ret = 1;
+ while (ret != 0) {
+ /* Load more input */
+ size_t srcSize = fread(src, 1, BUF_SIZE, in);
+ void* srcPtr = src;
+ void* srcEnd = srcPtr + srcSize;
+ if (srcSize == 0 || ferror(in)) {
+ printf("Decompress: not enough input or error reading file\n");
+ goto cleanup;
+ }
+ /* Allocate destination buffer if it isn't already */
+ if (!dst) {
+ LZ4F_frameInfo_t info;
+ ret = LZ4F_getFrameInfo(dctx, &info, src, &srcSize);
+ if (LZ4F_isError(ret)) {
+ printf("LZ4F_getFrameInfo error: %s\n", LZ4F_getErrorName(ret));
+ goto cleanup;
+ }
+ /* Allocating enough space for an entire block isn't necessary for
+ * correctness, but it allows some memcpy's to be elided.
+ */
+ dstCapacity = get_block_size(&info);
+ dst = malloc(dstCapacity);
if (!dst) { perror("decompress_file(dst)"); goto cleanup; }
- srcPtr += srcSize;
- srcSize = srcEnd - srcPtr;
- }
- /* Decompress:
- * Continue while there is more input to read and the frame isn't over.
- * If srcPtr == srcEnd then we know that there is no more output left in the
- * internal buffer left to flush.
- */
- while (srcPtr != srcEnd && ret != 0) {
- /* INVARIANT: Any data left in dst has already been written */
- size_t dstSize = dstCapacity;
- ret = LZ4F_decompress(dctx, dst, &dstSize, srcPtr, &srcSize, /* LZ4F_decompressOptions_t */ NULL);
- if (LZ4F_isError(ret)) {
- printf("Decompression error: %s\n", LZ4F_getErrorName(ret));
- goto cleanup;
- }
- /* Flush output */
- if (dstSize != 0){
- size_t written = fwrite(dst, 1, dstSize, out);
- printf("Writing %zu bytes\n", dstSize);
- if (written != dstSize) {
- printf("Decompress: Failed to write to file\n");
- goto cleanup;
- }
- }
- /* Update input */
- srcPtr += srcSize;
- srcSize = srcEnd - srcPtr;
- }
- }
- /* Check that there isn't trailing input data after the frame.
- * It is valid to have multiple frames in the same file, but this example
- * doesn't support it.
- */
- ret = fread(src, 1, 1, in);
- if (ret != 0 || !feof(in)) {
- printf("Decompress: Trailing data left in file after frame\n");
- goto cleanup;
- }
+ srcPtr += srcSize;
+ srcSize = srcEnd - srcPtr;
+ }
+ /* Decompress:
+ * Continue while there is more input to read and the frame isn't over.
+ * If srcPtr == srcEnd then we know that there is no more output left in the
+ * internal buffer left to flush.
+ */
+ while (srcPtr != srcEnd && ret != 0) {
+ /* INVARIANT: Any data left in dst has already been written */
+ size_t dstSize = dstCapacity;
+ ret = LZ4F_decompress(dctx, dst, &dstSize, srcPtr, &srcSize, /* LZ4F_decompressOptions_t */ NULL);
+ if (LZ4F_isError(ret)) {
+ printf("Decompression error: %s\n", LZ4F_getErrorName(ret));
+ goto cleanup;
+ }
+ /* Flush output */
+ if (dstSize != 0){
+ size_t written = fwrite(dst, 1, dstSize, out);
+ printf("Writing %zu bytes\n", dstSize);
+ if (written != dstSize) {
+ printf("Decompress: Failed to write to file\n");
+ goto cleanup;
+ }
+ }
+ /* Update input */
+ srcPtr += srcSize;
+ srcSize = srcEnd - srcPtr;
+ }
+ }
+ /* Check that there isn't trailing input data after the frame.
+ * It is valid to have multiple frames in the same file, but this example
+ * doesn't support it.
+ */
+ ret = fread(src, 1, 1, in);
+ if (ret != 0 || !feof(in)) {
+ printf("Decompress: Trailing data left in file after frame\n");
+ goto cleanup;
+ }
cleanup:
- free(src);
- free(dst);
- return LZ4F_freeDecompressionContext(dctx); /* note : free works on NULL */
+ free(src);
+ free(dst);
+ return LZ4F_freeDecompressionContext(dctx); /* note : free works on NULL */
}
int compare(FILE* fp0, FILE* fp1)
{
- int result = 0;
+ int result = 0;
- while(0 == result) {
- char b0[1024];
- char b1[1024];
- const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
- const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
+ while(0 == result) {
+ char b0[1024];
+ char b1[1024];
+ const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
+ const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
- result = (int) r0 - (int) r1;
+ result = (int) r0 - (int) r1;
- if (0 == r0 || 0 == r1) {
- break;
- }
- if (0 == result) {
- result = memcmp(b0, b1, r0);
- }
- }
+ if (0 == r0 || 0 == r1) {
+ break;
+ }
+ if (0 == result) {
+ result = memcmp(b0, b1, r0);
+ }
+ }
- return result;
+ return result;
}
int main(int argc, const char **argv) {
- char inpFilename[256] = { 0 };
- char lz4Filename[256] = { 0 };
- char decFilename[256] = { 0 };
-
- if(argc < 2) {
- printf("Please specify input filename\n");
- return 0;
- }
-
- snprintf(inpFilename, 256, "%s", argv[1]);
- snprintf(lz4Filename, 256, "%s.lz4", argv[1]);
- snprintf(decFilename, 256, "%s.lz4.dec", argv[1]);
-
- printf("inp = [%s]\n", inpFilename);
- printf("lz4 = [%s]\n", lz4Filename);
- printf("dec = [%s]\n", decFilename);
-
- /* compress */
- { FILE* const inpFp = fopen(inpFilename, "rb");
- FILE* const outFp = fopen(lz4Filename, "wb");
- size_t sizeIn = 0;
- size_t sizeOut = 0;
- size_t ret;
-
- printf("compress : %s -> %s\n", inpFilename, lz4Filename);
- ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
- if (ret) {
- printf("compress : failed with code %zu\n", ret);
- return ret;
- }
- printf("%s: %zu → %zu bytes, %.1f%%\n",
- inpFilename, sizeIn, sizeOut,
- (double)sizeOut / sizeIn * 100);
- printf("compress : done\n");
-
- fclose(outFp);
- fclose(inpFp);
- }
-
- /* decompress */
- { FILE* const inpFp = fopen(lz4Filename, "rb");
- FILE* const outFp = fopen(decFilename, "wb");
- size_t ret;
-
- printf("decompress : %s -> %s\n", lz4Filename, decFilename);
- ret = decompress_file(inpFp, outFp);
- if (ret) {
- printf("decompress : failed with code %zu\n", ret);
- return ret;
- }
- printf("decompress : done\n");
-
- fclose(outFp);
- fclose(inpFp);
- }
-
- /* verify */
- { FILE* const inpFp = fopen(inpFilename, "rb");
- FILE* const decFp = fopen(decFilename, "rb");
-
- printf("verify : %s <-> %s\n", inpFilename, decFilename);
- const int cmp = compare(inpFp, decFp);
- if(0 == cmp) {
- printf("verify : OK\n");
- } else {
- printf("verify : NG\n");
- }
-
- fclose(decFp);
- fclose(inpFp);
- }
+ char inpFilename[256] = { 0 };
+ char lz4Filename[256] = { 0 };
+ char decFilename[256] = { 0 };
+
+ if(argc < 2) {
+ printf("Please specify input filename\n");
+ return 0;
+ }
+
+ snprintf(inpFilename, 256, "%s", argv[1]);
+ snprintf(lz4Filename, 256, "%s.lz4", argv[1]);
+ snprintf(decFilename, 256, "%s.lz4.dec", argv[1]);
+
+ printf("inp = [%s]\n", inpFilename);
+ printf("lz4 = [%s]\n", lz4Filename);
+ printf("dec = [%s]\n", decFilename);
+
+ /* compress */
+ { FILE* const inpFp = fopen(inpFilename, "rb");
+ FILE* const outFp = fopen(lz4Filename, "wb");
+ size_t sizeIn = 0;
+ size_t sizeOut = 0;
+ size_t ret;
+
+ printf("compress : %s -> %s\n", inpFilename, lz4Filename);
+ ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
+ if (ret) {
+ printf("compress : failed with code %zu\n", ret);
+ return (int)ret;
+ }
+ printf("%s: %zu → %zu bytes, %.1f%%\n",
+ inpFilename, sizeIn, sizeOut,
+ (double)sizeOut / sizeIn * 100);
+ printf("compress : done\n");
+
+ fclose(outFp);
+ fclose(inpFp);
+ }
+
+ /* decompress */
+ { FILE* const inpFp = fopen(lz4Filename, "rb");
+ FILE* const outFp = fopen(decFilename, "wb");
+ size_t ret;
+
+ printf("decompress : %s -> %s\n", lz4Filename, decFilename);
+ ret = decompress_file(inpFp, outFp);
+ if (ret) {
+ printf("decompress : failed with code %zu\n", ret);
+ return (int)ret;
+ }
+ printf("decompress : done\n");
+
+ fclose(outFp);
+ fclose(inpFp);
+ }
+
+ /* verify */
+ { FILE* const inpFp = fopen(inpFilename, "rb");
+ FILE* const decFp = fopen(decFilename, "rb");
+
+ printf("verify : %s <-> %s\n", inpFilename, decFilename);
+ const int cmp = compare(inpFp, decFp);
+ if(0 == cmp) {
+ printf("verify : OK\n");
+ } else {
+ printf("verify : NG\n");
+ }
+
+ fclose(decFp);
+ fclose(inpFp);
+ }
}
diff --git a/lib/lz4.c b/lib/lz4.c
index 62e4dc2..41c0a28 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -945,6 +945,7 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
{
+ if (!LZ4_stream) return 0; /* support free on NULL */
FREEMEM(LZ4_stream);
return (0);
}
@@ -1284,11 +1285,6 @@ int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
/*===== streaming decompression functions =====*/
-/*
- * If you prefer dynamic allocation methods,
- * LZ4_createStreamDecode()
- * provides a pointer (void*) towards an initialized LZ4_streamDecode_t structure.
- */
LZ4_streamDecode_t* LZ4_createStreamDecode(void)
{
LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t));
@@ -1297,6 +1293,7 @@ LZ4_streamDecode_t* LZ4_createStreamDecode(void)
int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream)
{
+ if (!LZ4_stream) return 0; /* support free on NULL */
FREEMEM(LZ4_stream);
return 0;
}
diff --git a/lib/lz4.h b/lib/lz4.h
index 5b6bc92..86ca0d5 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -259,8 +259,8 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in
* Important : Previous data blocks are assumed to remain present and unmodified !
* 'dst' buffer must be already allocated.
* If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
- * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero.
- * After an error, the stream status is invalid, and it can only be reset or freed.
+ * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function @return==0.
+ * After an error, the stream status is invalid, it can only be reset or freed.
*/
LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
@@ -291,22 +291,21 @@ LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_str
*/
LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
-/*!
-LZ4_decompress_*_continue() :
- These decoding functions allow decompression of multiple blocks in "streaming" mode.
- Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
- In the case of a ring buffers, decoding buffer must be either :
- - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
- In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
- - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
- maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
- In which case, encoding and decoding buffers do not need to be synchronized,
- and encoding ring buffer can have any size, including small ones ( < 64 KB).
- - _At least_ 64 KB + 8 bytes + maxBlockSize.
- In which case, encoding and decoding buffers do not need to be synchronized,
- and encoding ring buffer can have any size, including larger than decoding buffer.
- Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
- and indicate where it is saved using LZ4_setStreamDecode()
+/*! LZ4_decompress_*_continue() :
+ * These decoding functions allow decompression of multiple blocks in "streaming" mode.
+ * Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
+ * In the case of a ring buffers, decoding buffer must be either :
+ * - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
+ * In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
+ * - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
+ * maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
+ * In which case, encoding and decoding buffers do not need to be synchronized,
+ * and encoding ring buffer can have any size, including small ones ( < 64 KB).
+ * - _At least_ 64 KB + 8 bytes + maxBlockSize.
+ * In which case, encoding and decoding buffers do not need to be synchronized,
+ * and encoding ring buffer can have any size, including larger than decoding buffer.
+ * Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
+ * and indicate where it is saved using LZ4_setStreamDecode()
*/
LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize);
@@ -380,7 +379,7 @@ typedef struct {
* init this structure before first use.
* note : only use in association with static linking !
* this definition is not API/ABI safe,
- * and may change in a future version !
+ * it may change in a future version !
*/
#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4)
#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long))
@@ -458,6 +457,7 @@ LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4
#endif /* LZ4_H_2983827168210 */
+
#if defined (__cplusplus)
}
#endif
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 994cc8b..e613901 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -61,13 +61,14 @@ You can contact the author at :
**************************************/
#include "lz4frame_static.h"
#include "lz4.h"
+#define LZ4_HC_STATIC_LINKING_ONLY
#include "lz4hc.h"
#define XXH_STATIC_LINKING_ONLY
#include "xxhash.h"
/*-************************************
-* Common Utils
+* Debug
**************************************/
#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
@@ -158,7 +159,7 @@ static void LZ4F_writeLE64 (void* dst, U64 value64)
#define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB
static const size_t minFHSize = 7;
-static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 15 */
+static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 19 */
static const size_t BHSize = 4;
@@ -170,6 +171,7 @@ typedef struct LZ4F_cctx_s
LZ4F_preferences_t prefs;
U32 version;
U32 cStage;
+ const LZ4F_CDict* cdict;
size_t maxBlockSize;
size_t maxBufferSize;
BYTE* tmpBuff;
@@ -178,7 +180,7 @@ typedef struct LZ4F_cctx_s
U64 totalInSize;
XXH32_state_t xxh;
void* lz4CtxPtr;
- U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */
+ U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */
} LZ4F_cctx_t;
@@ -216,6 +218,8 @@ static LZ4F_errorCode_t err0r(LZ4F_errorCodes code)
unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; }
+int LZ4F_compressionLevel_max(void) { return LZ4HC_CLEVEL_MAX; }
+
/*-************************************
* Private functions
@@ -256,11 +260,11 @@ static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSI
return requestedBSID;
}
-/* LZ4F_compressBound() :
- * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
- * prefsPtr is optional : if NULL is provided, preferences will be set to cover worst case scenario.
- * Result is always the same for a srcSize and prefsPtr, so it can be relied upon to size reusable buffers.
- * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
+/*! LZ4F_compressBound_internal() :
+ * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
+ * prefsPtr is optional : if NULL is provided, preferences will be set to cover worst case scenario.
+ * @return is always the same for a srcSize and prefsPtr, so it can be relied upon to size reusable buffers.
+ * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
*/
static size_t LZ4F_compressBound_internal(size_t srcSize,
const LZ4F_preferences_t* preferencesPtr,
@@ -271,8 +275,8 @@ static size_t LZ4F_compressBound_internal(size_t srcSize,
prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */
{ const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr;
U32 const flush = prefsPtr->autoFlush | (srcSize==0);
- LZ4F_blockSizeID_t const bid = prefsPtr->frameInfo.blockSizeID;
- size_t const blockSize = LZ4F_getBlockSize(bid);
+ LZ4F_blockSizeID_t const blockID = prefsPtr->frameInfo.blockSizeID;
+ size_t const blockSize = LZ4F_getBlockSize(blockID);
size_t const maxBuffered = blockSize - 1;
size_t const bufferedSize = MIN(alreadyBuffered, maxBuffered);
size_t const maxSrcSize = srcSize + bufferedSize;
@@ -281,17 +285,19 @@ static size_t LZ4F_compressBound_internal(size_t srcSize,
size_t const lastBlockSize = flush ? partialBlockSize : 0;
unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0);
- size_t const blockHeaderSize = 4; /* default, without block CRC option (which cannot be generated with current API) */
+ size_t const blockHeaderSize = 4;
+ size_t const blockCRCSize = 4 * prefsPtr->frameInfo.blockChecksumFlag;
size_t const frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4);
- return (blockHeaderSize * nbBlocks) + (blockSize * nbFullBlocks) + lastBlockSize + frameEnd;;
+ return ((blockHeaderSize + blockCRCSize) * nbBlocks) +
+ (blockSize * nbFullBlocks) + lastBlockSize + frameEnd;
}
}
size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
{
LZ4F_preferences_t prefs;
- size_t const headerSize = maxFHSize; /* max header size, including magic number and frame content size */
+ size_t const headerSize = maxFHSize; /* max header size, including optional fields */
if (preferencesPtr!=NULL) prefs = *preferencesPtr;
else memset(&prefs, 0, sizeof(prefs));
@@ -301,16 +307,19 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere
}
-/*! LZ4F_compressFrame() :
- * Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.0, in a single step.
- * The most important rule is that dstBuffer MUST be large enough (dstCapacity) to ensure compression completion even in worst case.
- * If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode)
- * Get the minimum value of dstCapacity by using LZ4F_compressFrameBound().
- * The LZ4F_preferences_t structure is optional : if NULL is provided as argument, preferences will be set to default.
- * The result of the function is the number of bytes written into dstBuffer.
- * The function outputs an error code if it fails (can be tested using LZ4F_isError())
+/*! LZ4F_compressFrame_usingCDict() :
+ * Compress srcBuffer using a dictionary, in a single step.
+ * cdict can be NULL, in which case, no dictionary is used.
+ * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
+ * The LZ4F_preferences_t structure is optional : you may provide NULL as argument,
+ * however, it's the only way to provide a dictID, so it's not recommended.
+ * @return : number of bytes written into dstBuffer,
+ * or an error code if it fails (can be tested using LZ4F_isError())
*/
-size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
+size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity,
+ const void* srcBuffer, size_t srcSize,
+ const LZ4F_CDict* cdict,
+ const LZ4F_preferences_t* preferencesPtr)
{
LZ4F_cctx_t cctxI;
LZ4_stream_t lz4ctx;
@@ -321,10 +330,8 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu
BYTE* const dstEnd = dstStart + dstCapacity;
memset(&cctxI, 0, sizeof(cctxI));
- memset(&options, 0, sizeof(options));
-
cctxI.version = LZ4F_VERSION;
- cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works because autoflush==1 & stableSrc==1 */
+ cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */
if (preferencesPtr!=NULL)
prefs = *preferencesPtr;
@@ -333,22 +340,23 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu
if (prefs.frameInfo.contentSize != 0)
prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */
- if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
- cctxI.lz4CtxPtr = &lz4ctx;
- cctxI.lz4CtxLevel = 1;
- } /* otherwise : will be created within LZ4F_compressBegin */
-
prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize);
prefs.autoFlush = 1;
if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID))
- prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* no need for linked blocks */
+ prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* only one block => no need for inter-block link */
+ if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
+ cctxI.lz4CtxPtr = &lz4ctx;
+ cctxI.lz4CtxLevel = 1;
+ } /* fast compression context pre-created on stack */
+
+ memset(&options, 0, sizeof(options));
options.stableSrc = 1;
- if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs)) /* condition to guarantee success */
+ if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs)) /* condition to guarantee success */
return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
- { size_t const headerSize = LZ4F_compressBegin(&cctxI, dstBuffer, dstCapacity, &prefs); /* write header */
+ { size_t const headerSize = LZ4F_compressBegin_usingCDict(&cctxI, dstBuffer, dstCapacity, cdict, &prefs); /* write header */
if (LZ4F_isError(headerSize)) return headerSize;
dstPtr += headerSize; /* header size */ }
@@ -360,24 +368,91 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu
if (LZ4F_isError(tailSize)) return tailSize;
dstPtr += tailSize; }
- if (prefs.compressionLevel >= LZ4HC_CLEVEL_MIN) /* Ctx allocation only for lz4hc */
+ if (prefs.compressionLevel >= LZ4HC_CLEVEL_MIN) /* Ctx allocation only for lz4hc */
FREEMEM(cctxI.lz4CtxPtr);
return (dstPtr - dstStart);
}
+/*! LZ4F_compressFrame() :
+ * Compress an entire srcBuffer into a valid LZ4 frame, in a single step.
+ * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
+ * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
+ * @return : number of bytes written into dstBuffer.
+ * or an error code if it fails (can be tested using LZ4F_isError())
+ */
+size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
+ const void* srcBuffer, size_t srcSize,
+ const LZ4F_preferences_t* preferencesPtr)
+{
+ return LZ4F_compressFrame_usingCDict(dstBuffer, dstCapacity,
+ srcBuffer, srcSize,
+ NULL, preferencesPtr);
+}
+
+
+/*-***************************************************
+* Dictionary compression
+*****************************************************/
+
+struct LZ4F_CDict_s {
+ void* dictContent;
+ LZ4_stream_t* fastCtx;
+ LZ4_streamHC_t* HCCtx;
+}; /* typedef'd to LZ4F_CDict within lz4frame_static.h */
+
+/*! LZ4F_createCDict() :
+ * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
+ * LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+ * LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
+ * `dictBuffer` can be released after LZ4F_CDict creation, since its content is copied within CDict
+ * @return : digested dictionary for compression, or NULL if failed */
+LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
+{
+ const char* dictStart = (const char*)dictBuffer;
+ LZ4F_CDict* cdict = (LZ4F_CDict*) malloc(sizeof(*cdict));
+ if (!cdict) return NULL;
+ if (dictSize > 64 KB) {
+ dictStart += dictSize - 64 KB;
+ dictSize = 64 KB;
+ }
+ cdict->dictContent = ALLOCATOR(dictSize);
+ cdict->fastCtx = LZ4_createStream();
+ cdict->HCCtx = LZ4_createStreamHC();
+ if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) {
+ LZ4F_freeCDict(cdict);
+ return NULL;
+ }
+ memcpy(cdict->dictContent, dictStart, dictSize);
+ LZ4_resetStream(cdict->fastCtx);
+ LZ4_loadDict (cdict->fastCtx, (const char*)cdict->dictContent, (int)dictSize);
+ LZ4_resetStreamHC(cdict->HCCtx, LZ4HC_CLEVEL_DEFAULT);
+ LZ4_loadDictHC(cdict->HCCtx, (const char*)cdict->dictContent, (int)dictSize);
+ return cdict;
+}
+
+void LZ4F_freeCDict(LZ4F_CDict* cdict)
+{
+ if (cdict==NULL) return; /* support free on NULL */
+ FREEMEM(cdict->dictContent);
+ LZ4_freeStream(cdict->fastCtx);
+ LZ4_freeStreamHC(cdict->HCCtx);
+ FREEMEM(cdict);
+}
+
+
/*-*********************************
* Advanced compression functions
***********************************/
/*! LZ4F_createCompressionContext() :
- * The first thing to do is to create a compressionContext object, which will be used in all compression operations.
- * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure.
- * The version provided MUST be LZ4F_VERSION. It is intended to track potential version differences between different binaries.
- * The function will provide a pointer to an allocated LZ4F_compressionContext_t object.
- * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation.
- * Object can release its memory using LZ4F_freeCompressionContext();
+ * The first thing to do is to create a compressionContext object, which will be used in all compression operations.
+ * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure.
+ * The version provided MUST be LZ4F_VERSION. It is intended to track potential incompatible differences between different binaries.
+ * The function will provide a pointer to an allocated LZ4F_compressionContext_t object.
+ * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation.
+ * Object can release its memory using LZ4F_freeCompressionContext();
*/
LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version)
{
@@ -385,7 +460,7 @@ LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_c
if (cctxPtr==NULL) return err0r(LZ4F_ERROR_allocation_failed);
cctxPtr->version = version;
- cctxPtr->cStage = 0; /* Next stage : write header */
+ cctxPtr->cStage = 0; /* Next stage : init stream */
*LZ4F_compressionContextPtr = (LZ4F_compressionContext_t)cctxPtr;
@@ -397,8 +472,8 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
{
LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)LZ4F_compressionContext;
- if (cctxPtr != NULL) { /* null pointers can be safely provided to this function, like free() */
- FREEMEM(cctxPtr->lz4CtxPtr);
+ if (cctxPtr != NULL) { /* support free on NULL */
+ FREEMEM(cctxPtr->lz4CtxPtr); /* works because LZ4_streamHC_t and LZ4_stream_t are simple POD types */
FREEMEM(cctxPtr->tmpBuff);
FREEMEM(LZ4F_compressionContext);
}
@@ -407,22 +482,23 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
}
-/*! LZ4F_compressBegin() :
- * will write the frame header into dstBuffer.
- * dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_HEADER_SIZE_MAX bytes.
- * @return : number of bytes written into dstBuffer for the header
- * or an error code (can be tested using LZ4F_isError())
+/*! LZ4F_compressBegin_usingCDict() :
+ * init streaming compression and writes frame header into dstBuffer.
+ * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes.
+ * @return : number of bytes written into dstBuffer for the header
+ * or an error code (can be tested using LZ4F_isError())
*/
-size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* preferencesPtr)
+size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
+ void* dstBuffer, size_t dstCapacity,
+ const LZ4F_CDict* cdict,
+ const LZ4F_preferences_t* preferencesPtr)
{
LZ4F_preferences_t prefNull;
BYTE* const dstStart = (BYTE*)dstBuffer;
BYTE* dstPtr = dstStart;
BYTE* headerStart;
- size_t requiredBuffSize;
if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
- if (cctxPtr->cStage != 0) return err0r(LZ4F_ERROR_GENERIC);
memset(&prefNull, 0, sizeof(prefNull));
if (preferencesPtr == NULL) preferencesPtr = &prefNull;
cctxPtr->prefs = *preferencesPtr;
@@ -437,31 +513,46 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit
cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC();
if (cctxPtr->lz4CtxPtr == NULL) return err0r(LZ4F_ERROR_allocation_failed);
cctxPtr->lz4CtxLevel = tableID;
- }
- }
+ } }
/* Buffer Management */
- if (cctxPtr->prefs.frameInfo.blockSizeID == 0) cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT;
+ if (cctxPtr->prefs.frameInfo.blockSizeID == 0)
+ cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT;
cctxPtr->maxBlockSize = LZ4F_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID);
- requiredBuffSize = cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 128 KB);
- if (preferencesPtr->autoFlush)
- requiredBuffSize = (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 64 KB; /* just needs dict */
+ { size_t const requiredBuffSize = preferencesPtr->autoFlush ?
+ (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 64 KB : /* only needs windows size */
+ cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 128 KB);
- if (cctxPtr->maxBufferSize < requiredBuffSize) {
- cctxPtr->maxBufferSize = 0;
- FREEMEM(cctxPtr->tmpBuff);
- cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize);
- if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed);
- cctxPtr->maxBufferSize = requiredBuffSize;
- }
+ if (cctxPtr->maxBufferSize < requiredBuffSize) {
+ cctxPtr->maxBufferSize = 0;
+ FREEMEM(cctxPtr->tmpBuff);
+ cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize);
+ if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed);
+ cctxPtr->maxBufferSize = requiredBuffSize;
+ } }
cctxPtr->tmpIn = cctxPtr->tmpBuff;
cctxPtr->tmpInSize = 0;
XXH32_reset(&(cctxPtr->xxh), 0);
- if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
- LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr));
- else
- LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel);
+
+ /* context init */
+ cctxPtr->cdict = cdict;
+ if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) {
+ /* frame init only for blockLinked : blockIndependent will be init at each block */
+ if (cdict) {
+ if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
+ memcpy(cctxPtr->lz4CtxPtr, cdict->fastCtx, sizeof(*cdict->fastCtx));
+ } else {
+ memcpy(cctxPtr->lz4CtxPtr, cdict->HCCtx, sizeof(*cdict->HCCtx));
+ LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+ }
+ } else {
+ if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
+ LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr));
+ else
+ LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel);
+ }
+ }
/* Magic Number */
LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER);
@@ -470,9 +561,11 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit
/* FLG Byte */
*dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */
- + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) /* Block mode */
- + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) /* Frame checksum */
- + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3)); /* Frame content size */
+ + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5)
+ + ((cctxPtr->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4)
+ + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3)
+ + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2)
+ + (cctxPtr->prefs.frameInfo.dictID > 0) );
/* BD Byte */
*dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4);
/* Optional Frame content size field */
@@ -481,16 +574,36 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit
dstPtr += 8;
cctxPtr->totalInSize = 0;
}
- /* CRC Byte */
+ /* Optional dictionary ID field */
+ if (cctxPtr->prefs.frameInfo.dictID) {
+ LZ4F_writeLE32(dstPtr, cctxPtr->prefs.frameInfo.dictID);
+ dstPtr += 4;
+ }
+ /* Header CRC Byte */
*dstPtr = LZ4F_headerChecksum(headerStart, dstPtr - headerStart);
dstPtr++;
cctxPtr->cStage = 1; /* header written, now request input data block */
-
return (dstPtr - dstStart);
}
+/*! LZ4F_compressBegin() :
+ * init streaming compression and writes frame header into dstBuffer.
+ * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes.
+ * preferencesPtr can be NULL, in which case default parameters are selected.
+ * @return : number of bytes written into dstBuffer for the header
+ * or an error code (can be tested using LZ4F_isError())
+ */
+size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr,
+ void* dstBuffer, size_t dstCapacity,
+ const LZ4F_preferences_t* preferencesPtr)
+{
+ return LZ4F_compressBegin_usingCDict(cctxPtr, dstBuffer, dstCapacity,
+ NULL, preferencesPtr);
+}
+
+
/* LZ4F_compressBound() :
* @ return size of Dst buffer given a srcSize to handle worst case situations.
* The LZ4F_frameInfo_t structure is optional : if NULL, preferences will be set to cover worst case situations.
@@ -502,49 +615,75 @@ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesP
}
-typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level);
+typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level, const LZ4F_CDict* cdict);
-static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx, int level)
+
+/*! LZ4F_makeBlock():
+ * compress a single block, add header and checksum
+ * assumption : dst buffer capacity is >= srcSize */
+static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize,
+ compressFunc_t compress, void* lz4ctx, int level,
+ const LZ4F_CDict* cdict, LZ4F_blockChecksum_t crcFlag)
{
- /* compress a single block */
BYTE* const cSizePtr = (BYTE*)dst;
- U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1), level);
+ U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4),
+ (int)(srcSize), (int)(srcSize-1),
+ level, cdict);
LZ4F_writeLE32(cSizePtr, cSize);
if (cSize == 0) { /* compression failed */
cSize = (U32)srcSize;
LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG);
memcpy(cSizePtr+4, src, srcSize);
}
- return cSize + 4;
+ if (crcFlag) {
+ U32 const crc32 = XXH32(cSizePtr+4, cSize, 0); /* checksum of compressed data */
+ LZ4F_writeLE32(cSizePtr+4+cSize, crc32);
+ }
+ return 4 + cSize + ((U32)crcFlag)*4;
}
-static int LZ4F_localLZ4_compress_limitedOutput_withState(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level)
+static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
{
int const acceleration = (level < -1) ? -level : 1;
+ if (cdict) {
+ memcpy(ctx, cdict->fastCtx, sizeof(*cdict->fastCtx));
+ return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
+ }
return LZ4_compress_fast_extState(ctx, src, dst, srcSize, dstCapacity, acceleration);
}
-static int LZ4F_localLZ4_compress_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level)
+static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
{
int const acceleration = (level < -1) ? -level : 1;
+ (void)cdict; /* init once at beginning of frame */
return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
}
-static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level)
+static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
{
- (void) level;
- return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstSize);
+ if (cdict) {
+ memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx));
+ LZ4_setCompressionLevel((LZ4_streamHC_t*)ctx, level);
+ return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
+ }
+ return LZ4_compress_HC_extStateHC(ctx, src, dst, srcSize, dstCapacity, level);
+}
+
+static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
+{
+ (void)level; (void)cdict; /* init once at beginning of frame */
+ return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
}
static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level)
{
if (level < LZ4HC_CLEVEL_MIN) {
- if (blockMode == LZ4F_blockIndependent) return LZ4F_localLZ4_compress_limitedOutput_withState;
- return LZ4F_localLZ4_compress_limitedOutput_continue;
+ if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlock;
+ return LZ4F_compressBlock_continue;
}
- if (blockMode == LZ4F_blockIndependent) return LZ4_compress_HC_extStateHC;
- return LZ4F_localLZ4_compressHC_limitedOutput_continue;
+ if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlockHC;
+ return LZ4F_compressBlockHC_continue;
}
static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
@@ -557,15 +696,16 @@ static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
/*! LZ4F_compressUpdate() :
-* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
-* The most important rule is that dstBuffer MUST be large enough (dstCapacity) to ensure compression completion even in worst case.
-* If this condition is not respected, LZ4F_compress() will fail (result is an errorCode)
-* You can get the minimum value of dstCapacity by using LZ4F_compressBound()
-* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
-* The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered.
-* The function outputs an error code if it fails (can be tested using LZ4F_isError())
-*/
-size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr)
+ * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
+ * dstBuffer MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
+ * LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
+ * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
+ * or an error code if it fails (which can be tested using LZ4F_isError())
+ */
+size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
+ void* dstBuffer, size_t dstCapacity,
+ const void* srcBuffer, size_t srcSize,
+ const LZ4F_compressOptions_t* compressOptionsPtr)
{
LZ4F_compressOptions_t cOptionsNull;
size_t const blockSize = cctxPtr->maxBlockSize;
@@ -597,7 +737,9 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci
memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy);
srcPtr += sizeToCopy;
- dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+ dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, blockSize,
+ compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
+ cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
cctxPtr->tmpInSize = 0;
@@ -605,16 +747,20 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci
}
while ((size_t)(srcEnd - srcPtr) >= blockSize) {
- /* compress full block */
+ /* compress full blocks */
lastBlockCompressed = fromSrcBuffer;
- dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+ dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, blockSize,
+ compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
+ cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
srcPtr += blockSize;
}
if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) {
/* compress remaining input < blockSize */
lastBlockCompressed = fromSrcBuffer;
- dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+ dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, srcEnd - srcPtr,
+ compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
+ cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
srcPtr = srcEnd;
}
@@ -676,7 +822,9 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const
compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
/* compress tmp buffer */
- dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+ dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize,
+ compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
+ cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize;
cctxPtr->tmpInSize = 0;
@@ -733,10 +881,23 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize,
* Frame Decompression
*****************************************************/
+typedef enum {
+ dstage_getFrameHeader=0, dstage_storeFrameHeader,
+ dstage_init,
+ dstage_getBlockHeader, dstage_storeBlockHeader,
+ dstage_copyDirect, dstage_getBlockChecksum,
+ dstage_getCBlock, dstage_storeCBlock,
+ dstage_decodeCBlock, dstage_decodeCBlock_intoDst,
+ dstage_decodeCBlock_intoTmp, dstage_flushOut,
+ dstage_getSuffix, dstage_storeSuffix,
+ dstage_getSFrameSize, dstage_storeSFrameSize,
+ dstage_skipSkippable
+} dStage_t;
+
struct LZ4F_dctx_s {
LZ4F_frameInfo_t frameInfo;
U32 version;
- U32 dStage;
+ dStage_t dStage;
U64 frameRemainingSize;
size_t maxBlockSize;
size_t maxBufferSize;
@@ -744,40 +905,41 @@ struct LZ4F_dctx_s {
size_t tmpInSize;
size_t tmpInTarget;
BYTE* tmpOutBuffer;
- const BYTE* dict;
+ const BYTE* dict;
size_t dictSize;
BYTE* tmpOut;
size_t tmpOutSize;
size_t tmpOutStart;
XXH32_state_t xxh;
+ XXH32_state_t blockChecksum;
BYTE header[16];
}; /* typedef'd to LZ4F_dctx in lz4frame.h */
/*! LZ4F_createDecompressionContext() :
- * Create a decompressionContext object, which will track all decompression operations.
- * Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object.
- * Object can later be released using LZ4F_freeDecompressionContext().
- * @return : if != 0, there was an error during context creation.
+ * Create a decompressionContext object, which will track all decompression operations.
+ * Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object.
+ * Object can later be released using LZ4F_freeDecompressionContext().
+ * @return : if != 0, there was an error during context creation.
*/
LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber)
{
- LZ4F_dctx* const dctxPtr = (LZ4F_dctx*)ALLOCATOR(sizeof(LZ4F_dctx));
- if (dctxPtr==NULL) return err0r(LZ4F_ERROR_GENERIC);
+ LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOCATOR(sizeof(LZ4F_dctx));
+ if (dctx==NULL) return err0r(LZ4F_ERROR_GENERIC);
- dctxPtr->version = versionNumber;
- *LZ4F_decompressionContextPtr = dctxPtr;
+ dctx->version = versionNumber;
+ *LZ4F_decompressionContextPtr = dctx;
return LZ4F_OK_NoError;
}
-LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctxPtr)
+LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx)
{
LZ4F_errorCode_t result = LZ4F_OK_NoError;
- if (dctxPtr != NULL) { /* can accept NULL input, like free() */
- result = (LZ4F_errorCode_t)dctxPtr->dStage;
- FREEMEM(dctxPtr->tmpIn);
- FREEMEM(dctxPtr->tmpOutBuffer);
- FREEMEM(dctxPtr);
+ if (dctx != NULL) { /* can accept NULL input, like free() */
+ result = (LZ4F_errorCode_t)dctx->dStage;
+ FREEMEM(dctx->tmpIn);
+ FREEMEM(dctx->tmpOutBuffer);
+ FREEMEM(dctx);
}
return result;
}
@@ -785,22 +947,11 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctxPtr)
/*==--- Streaming Decompression operations ---==*/
-typedef enum {
- dstage_getHeader=0, dstage_storeHeader,
- dstage_init,
- dstage_getCBlockSize, dstage_storeCBlockSize,
- dstage_copyDirect,
- dstage_getCBlock, dstage_storeCBlock,
- dstage_decodeCBlock, dstage_decodeCBlock_intoDst,
- dstage_decodeCBlock_intoTmp, dstage_flushOut,
- dstage_getSuffix, dstage_storeSuffix,
- dstage_getSFrameSize, dstage_storeSFrameSize,
- dstage_skipSkippable
-} dStage_t;
-
void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx)
{
- dctx->dStage = dstage_getHeader;
+ dctx->dStage = dstage_getFrameHeader;
+ dctx->dict = NULL;
+ dctx->dictSize = 0;
}
@@ -817,12 +968,14 @@ static size_t LZ4F_headerSize(const void* src, size_t srcSize)
if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) return 8;
/* control magic number */
- if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER) return err0r(LZ4F_ERROR_frameType_unknown);
+ if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER)
+ return err0r(LZ4F_ERROR_frameType_unknown);
/* Frame Header Size */
{ BYTE const FLG = ((const BYTE*)src)[4];
U32 const contentSizeFlag = (FLG>>3) & _1BIT;
- return contentSizeFlag ? maxFHSize : minFHSize;
+ U32 const dictIDFlag = FLG & _1BIT;
+ return minFHSize + (contentSizeFlag*8) + (dictIDFlag*4);
}
}
@@ -830,62 +983,63 @@ static size_t LZ4F_headerSize(const void* src, size_t srcSize)
/*! LZ4F_decodeHeader() :
* input : `src` points at the **beginning of the frame**
* output : set internal values of dctx, such as
- * dctxPtr->frameInfo and dctxPtr->dStage.
+ * dctx->frameInfo and dctx->dStage.
* Also allocates internal buffers.
* @return : nb Bytes read from src (necessarily <= srcSize)
* or an error code (testable with LZ4F_isError())
*/
-static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcSize)
+static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize)
{
- unsigned blockMode, contentSizeFlag, contentChecksumFlag, blockSizeID;
+ unsigned blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, dictIDFlag, blockSizeID;
size_t frameHeaderSize;
const BYTE* srcPtr = (const BYTE*)src;
/* need to decode header to get frameInfo */
if (srcSize < minFHSize) return err0r(LZ4F_ERROR_frameHeader_incomplete); /* minimal frame header size */
- memset(&(dctxPtr->frameInfo), 0, sizeof(dctxPtr->frameInfo));
+ memset(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo));
/* special case : skippable frames */
if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) {
- dctxPtr->frameInfo.frameType = LZ4F_skippableFrame;
- if (src == (void*)(dctxPtr->header)) {
- dctxPtr->tmpInSize = srcSize;
- dctxPtr->tmpInTarget = 8;
- dctxPtr->dStage = dstage_storeSFrameSize;
+ dctx->frameInfo.frameType = LZ4F_skippableFrame;
+ if (src == (void*)(dctx->header)) {
+ dctx->tmpInSize = srcSize;
+ dctx->tmpInTarget = 8;
+ dctx->dStage = dstage_storeSFrameSize;
return srcSize;
} else {
- dctxPtr->dStage = dstage_getSFrameSize;
+ dctx->dStage = dstage_getSFrameSize;
return 4;
}
}
/* control magic number */
- if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) return err0r(LZ4F_ERROR_frameType_unknown);
- dctxPtr->frameInfo.frameType = LZ4F_frame;
+ if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER)
+ return err0r(LZ4F_ERROR_frameType_unknown);
+ dctx->frameInfo.frameType = LZ4F_frame;
/* Flags */
{ U32 const FLG = srcPtr[4];
U32 const version = (FLG>>6) & _2BITS;
- U32 const blockChecksumFlag = (FLG>>4) & _1BIT;
+ blockChecksumFlag = (FLG>>4) & _1BIT;
blockMode = (FLG>>5) & _1BIT;
contentSizeFlag = (FLG>>3) & _1BIT;
contentChecksumFlag = (FLG>>2) & _1BIT;
+ dictIDFlag = FLG & _1BIT;
/* validate */
- if (((FLG>>0)&_2BITS) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bits */
+ if (((FLG>>1)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bit */
if (version != 1) return err0r(LZ4F_ERROR_headerVersion_wrong); /* Version Number, only supported value */
- if (blockChecksumFlag != 0) return err0r(LZ4F_ERROR_blockChecksum_unsupported); /* Not supported for the time being */
}
/* Frame Header Size */
- frameHeaderSize = contentSizeFlag ? maxFHSize : minFHSize;
+ frameHeaderSize = minFHSize + (contentSizeFlag*8) + (dictIDFlag*4);
if (srcSize < frameHeaderSize) {
/* not enough input to fully decode frame header */
- if (srcPtr != dctxPtr->header)
- memcpy(dctxPtr->header, srcPtr, srcSize);
- dctxPtr->tmpInSize = srcSize;
- dctxPtr->tmpInTarget = frameHeaderSize;
- dctxPtr->dStage = dstage_storeHeader;
+ if (srcPtr != dctx->header)
+ memcpy(dctx->header, srcPtr, srcSize);
+ dctx->tmpInSize = srcSize;
+ dctx->tmpInTarget = frameHeaderSize;
+ dctx->dStage = dstage_storeFrameHeader;
return srcSize;
}
@@ -898,50 +1052,56 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcS
}
/* check header */
- { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5);
- if (HC != srcPtr[frameHeaderSize-1]) return err0r(LZ4F_ERROR_headerChecksum_invalid); }
+ { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5);
+ if (HC != srcPtr[frameHeaderSize-1])
+ return err0r(LZ4F_ERROR_headerChecksum_invalid);
+ }
/* save */
- dctxPtr->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode;
- dctxPtr->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag;
- dctxPtr->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID;
- dctxPtr->maxBlockSize = LZ4F_getBlockSize(blockSizeID);
+ dctx->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode;
+ dctx->frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)blockChecksumFlag;
+ dctx->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag;
+ dctx->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID;
+ dctx->maxBlockSize = LZ4F_getBlockSize(blockSizeID);
if (contentSizeFlag)
- dctxPtr->frameRemainingSize = dctxPtr->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6);
+ dctx->frameRemainingSize =
+ dctx->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6);
+ if (dictIDFlag)
+ dctx->frameInfo.dictID = LZ4F_readLE32(srcPtr + frameHeaderSize - 5);
- dctxPtr->dStage = dstage_init;
+ dctx->dStage = dstage_init;
return frameHeaderSize;
}
/*! LZ4F_getFrameInfo() :
- * This function extracts frame parameters (max blockSize, frame checksum, etc.).
- * Usage is optional. Objective is to provide relevant information for allocation purposes.
- * This function works in 2 situations :
+ * This function extracts frame parameters (max blockSize, frame checksum, etc.).
+ * Usage is optional. Objective is to provide relevant information for allocation purposes.
+ * This function works in 2 situations :
* - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process.
* Amount of input data provided must be large enough to successfully decode the frame header.
* A header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes. It's possible to provide more input data than this minimum.
* - After decoding has been started. In which case, no input is read, frame parameters are extracted from dctx.
- * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
- * Decompression must resume from (srcBuffer + *srcSizePtr).
+ * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
+ * Decompression must resume from (srcBuffer + *srcSizePtr).
* @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
* or an error code which can be tested using LZ4F_isError()
- * note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped.
- * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
+ * note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped.
+ * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
*/
-LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctxPtr, LZ4F_frameInfo_t* frameInfoPtr,
+LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoPtr,
const void* srcBuffer, size_t* srcSizePtr)
{
- if (dctxPtr->dStage > dstage_storeHeader) { /* assumption : dstage_* header enum at beginning of range */
+ if (dctx->dStage > dstage_storeFrameHeader) { /* assumption : dstage_* header enum at beginning of range */
/* frameInfo already decoded */
size_t o=0, i=0;
*srcSizePtr = 0;
- *frameInfoPtr = dctxPtr->frameInfo;
+ *frameInfoPtr = dctx->frameInfo;
/* returns : recommended nb of bytes for LZ4F_decompress() */
- return LZ4F_decompress(dctxPtr, NULL, &o, NULL, &i, NULL);
+ return LZ4F_decompress(dctx, NULL, &o, NULL, &i, NULL);
} else {
- if (dctxPtr->dStage == dstage_storeHeader) {
+ if (dctx->dStage == dstage_storeFrameHeader) {
/* frame decoding already started, in the middle of header => automatic fail */
*srcSizePtr = 0;
return err0r(LZ4F_ERROR_frameDecoding_alreadyStarted);
@@ -954,109 +1114,99 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctxPtr, LZ4F_frameInfo_t* frameIn
return err0r(LZ4F_ERROR_frameHeader_incomplete);
}
- decodeResult = LZ4F_decodeHeader(dctxPtr, srcBuffer, hSize);
+ decodeResult = LZ4F_decodeHeader(dctx, srcBuffer, hSize);
if (LZ4F_isError(decodeResult)) {
*srcSizePtr = 0;
} else {
*srcSizePtr = decodeResult;
decodeResult = BHSize; /* block header size */
}
- *frameInfoPtr = dctxPtr->frameInfo;
+ *frameInfoPtr = dctx->frameInfo;
return decodeResult;
} }
}
-/* trivial redirector, for common prototype */
-static int LZ4F_decompress_safe (const char* src,
- char* dst,
- int compressedSize,
- int dstCapacity,
- const char* dictStart,
- int dictSize)
+/* LZ4F_updateDict() :
+ * only used for LZ4F_blockLinked mode */
+static void LZ4F_updateDict(LZ4F_dctx* dctx, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp)
{
- (void)dictStart; (void)dictSize;
- return LZ4_decompress_safe (src, dst, compressedSize, dstCapacity);
-}
-
+ if (dctx->dictSize==0)
+ dctx->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */
-static void LZ4F_updateDict(LZ4F_dctx* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp)
-{
- if (dctxPtr->dictSize==0)
- dctxPtr->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */
-
- if (dctxPtr->dict + dctxPtr->dictSize == dstPtr) { /* dictionary continuity */
- dctxPtr->dictSize += dstSize;
+ if (dctx->dict + dctx->dictSize == dstPtr) { /* dictionary continuity */
+ dctx->dictSize += dstSize;
return;
}
if (dstPtr - dstPtr0 + dstSize >= 64 KB) { /* dstBuffer large enough to become dictionary */
- dctxPtr->dict = (const BYTE*)dstPtr0;
- dctxPtr->dictSize = dstPtr - dstPtr0 + dstSize;
+ dctx->dict = (const BYTE*)dstPtr0;
+ dctx->dictSize = dstPtr - dstPtr0 + dstSize;
return;
}
- if ((withinTmp) && (dctxPtr->dict == dctxPtr->tmpOutBuffer)) {
- /* assumption : dctxPtr->dict + dctxPtr->dictSize == dctxPtr->tmpOut + dctxPtr->tmpOutStart */
- dctxPtr->dictSize += dstSize;
+ if ((withinTmp) && (dctx->dict == dctx->tmpOutBuffer)) {
+ /* assumption : dctx->dict + dctx->dictSize == dctx->tmpOut + dctx->tmpOutStart */
+ dctx->dictSize += dstSize;
return;
}
if (withinTmp) { /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */
- size_t const preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer;
- size_t copySize = 64 KB - dctxPtr->tmpOutSize;
- const BYTE* const oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart;
- if (dctxPtr->tmpOutSize > 64 KB) copySize = 0;
+ size_t const preserveSize = dctx->tmpOut - dctx->tmpOutBuffer;
+ size_t copySize = 64 KB - dctx->tmpOutSize;
+ const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart;
+ if (dctx->tmpOutSize > 64 KB) copySize = 0;
if (copySize > preserveSize) copySize = preserveSize;
- memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize);
+ memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize);
- dctxPtr->dict = dctxPtr->tmpOutBuffer;
- dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart + dstSize;
+ dctx->dict = dctx->tmpOutBuffer;
+ dctx->dictSize = preserveSize + dctx->tmpOutStart + dstSize;
return;
}
- if (dctxPtr->dict == dctxPtr->tmpOutBuffer) { /* copy dst into tmp to complete dict */
- if (dctxPtr->dictSize + dstSize > dctxPtr->maxBufferSize) { /* tmp buffer not large enough */
+ if (dctx->dict == dctx->tmpOutBuffer) { /* copy dst into tmp to complete dict */
+ if (dctx->dictSize + dstSize > dctx->maxBufferSize) { /* tmp buffer not large enough */
size_t const preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */
- memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize);
- dctxPtr->dictSize = preserveSize;
+ memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize);
+ dctx->dictSize = preserveSize;
}
- memcpy(dctxPtr->tmpOutBuffer + dctxPtr->dictSize, dstPtr, dstSize);
- dctxPtr->dictSize += dstSize;
+ memcpy(dctx->tmpOutBuffer + dctx->dictSize, dstPtr, dstSize);
+ dctx->dictSize += dstSize;
return;
}
/* join dict & dest into tmp */
{ size_t preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */
- if (preserveSize > dctxPtr->dictSize) preserveSize = dctxPtr->dictSize;
- memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize);
- memcpy(dctxPtr->tmpOutBuffer + preserveSize, dstPtr, dstSize);
- dctxPtr->dict = dctxPtr->tmpOutBuffer;
- dctxPtr->dictSize = preserveSize + dstSize;
+ if (preserveSize > dctx->dictSize) preserveSize = dctx->dictSize;
+ memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize);
+ memcpy(dctx->tmpOutBuffer + preserveSize, dstPtr, dstSize);
+ dctx->dict = dctx->tmpOutBuffer;
+ dctx->dictSize = preserveSize + dstSize;
}
}
/*! LZ4F_decompress() :
- * Call this function repetitively to regenerate data compressed within srcBuffer.
- * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
+ * Call this function repetitively to regenerate compressed data in srcBuffer.
+ * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer
+ * into dstBuffer of capacity *dstSizePtr.
*
- * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
+ * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
*
- * The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
- * If the number of bytes read is < number of bytes provided, then the decompression operation is not complete.
- * Remaining data will have to be presented again in a subsequent invocation.
+ * The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
+ * If number of bytes read is < number of bytes provided, then decompression operation is not complete.
+ * Remaining data will have to be presented again in a subsequent invocation.
*
- * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress.
- * Basically, it's the size of the current (or remaining) compressed block + header of next block.
- * Respecting the hint provides some boost to performance, since it allows less buffer shuffling.
- * Note that this is just a hint, it's always possible to any srcSize value.
- * When a frame is fully decoded, @return will be 0.
- * If decompression failed, @return is an error code which can be tested using LZ4F_isError().
+ * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress.
+ * Schematically, it's the size of the current (or remaining) compressed block + header of next block.
+ * Respecting the hint provides a small boost to performance, since it allows less buffer shuffling.
+ * Note that this is just a hint, and it's always possible to any srcSize value.
+ * When a frame is fully decoded, @return will be 0.
+ * If decompression failed, @return is an error code which can be tested using LZ4F_isError().
*/
-size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
+size_t LZ4F_decompress(LZ4F_dctx* dctx,
void* dstBuffer, size_t* dstSizePtr,
const void* srcBuffer, size_t* srcSizePtr,
const LZ4F_decompressOptions_t* decompressOptionsPtr)
@@ -1078,257 +1228,288 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
*srcSizePtr = 0;
*dstSizePtr = 0;
- /* behaves like a state machine */
+ /* behaves as a state machine */
while (doAnotherStage) {
- switch(dctxPtr->dStage)
+ switch(dctx->dStage)
{
- case dstage_getHeader:
+ case dstage_getFrameHeader:
if ((size_t)(srcEnd-srcPtr) >= maxFHSize) { /* enough to decode - shortcut */
- LZ4F_errorCode_t const hSize = LZ4F_decodeHeader(dctxPtr, srcPtr, srcEnd-srcPtr); /* will change dStage appropriately */
+ size_t const hSize = LZ4F_decodeHeader(dctx, srcPtr, srcEnd-srcPtr); /* will update dStage appropriately */
if (LZ4F_isError(hSize)) return hSize;
srcPtr += hSize;
break;
}
- dctxPtr->tmpInSize = 0;
+ dctx->tmpInSize = 0;
if (srcEnd-srcPtr == 0) return minFHSize; /* 0-size input */
- dctxPtr->tmpInTarget = minFHSize; /* minimum to attempt decode */
- dctxPtr->dStage = dstage_storeHeader;
+ dctx->tmpInTarget = minFHSize; /* minimum to attempt decode */
+ dctx->dStage = dstage_storeFrameHeader;
/* fall-through */
- case dstage_storeHeader:
- { size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize;
- if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr;
- memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
- dctxPtr->tmpInSize += sizeToCopy;
+ case dstage_storeFrameHeader:
+ { size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, (size_t)(srcEnd - srcPtr));
+ memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);
+ dctx->tmpInSize += sizeToCopy;
srcPtr += sizeToCopy;
- if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) {
- nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + BHSize; /* rest of header + nextBlockHeader */
+ if (dctx->tmpInSize < dctx->tmpInTarget) {
+ nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize; /* rest of header + nextBlockHeader */
doAnotherStage = 0; /* not enough src data, ask for some more */
break;
}
- { LZ4F_errorCode_t const hSize = LZ4F_decodeHeader(dctxPtr, dctxPtr->header, dctxPtr->tmpInTarget); /* will change dStage appropriately */
+ { size_t const hSize = LZ4F_decodeHeader(dctx, dctx->header, dctx->tmpInTarget); /* will update dStage appropriately */
if (LZ4F_isError(hSize)) return hSize;
}
break;
}
case dstage_init:
- if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_reset(&(dctxPtr->xxh), 0);
+ if (dctx->frameInfo.contentChecksumFlag) XXH32_reset(&(dctx->xxh), 0);
/* internal buffers allocation */
- { size_t const bufferNeeded = dctxPtr->maxBlockSize + ((dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB);
- if (bufferNeeded > dctxPtr->maxBufferSize) { /* tmp buffers too small */
- dctxPtr->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/
- FREEMEM(dctxPtr->tmpIn);
- dctxPtr->tmpIn = (BYTE*)ALLOCATOR(dctxPtr->maxBlockSize);
- if (dctxPtr->tmpIn == NULL) return err0r(LZ4F_ERROR_allocation_failed);
- FREEMEM(dctxPtr->tmpOutBuffer);
- dctxPtr->tmpOutBuffer= (BYTE*)ALLOCATOR(bufferNeeded);
- if (dctxPtr->tmpOutBuffer== NULL) return err0r(LZ4F_ERROR_allocation_failed);
- dctxPtr->maxBufferSize = bufferNeeded;
+ { size_t const bufferNeeded = dctx->maxBlockSize + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB) + 4 /* block checksum */;
+ if (bufferNeeded > dctx->maxBufferSize) { /* tmp buffers too small */
+ dctx->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/
+ FREEMEM(dctx->tmpIn);
+ dctx->tmpIn = (BYTE*)ALLOCATOR(dctx->maxBlockSize);
+ if (dctx->tmpIn == NULL) return err0r(LZ4F_ERROR_allocation_failed);
+ FREEMEM(dctx->tmpOutBuffer);
+ dctx->tmpOutBuffer= (BYTE*)ALLOCATOR(bufferNeeded);
+ if (dctx->tmpOutBuffer== NULL) return err0r(LZ4F_ERROR_allocation_failed);
+ dctx->maxBufferSize = bufferNeeded;
} }
- dctxPtr->tmpInSize = 0;
- dctxPtr->tmpInTarget = 0;
- dctxPtr->dict = dctxPtr->tmpOutBuffer;
- dctxPtr->dictSize = 0;
- dctxPtr->tmpOut = dctxPtr->tmpOutBuffer;
- dctxPtr->tmpOutStart = 0;
- dctxPtr->tmpOutSize = 0;
-
- dctxPtr->dStage = dstage_getCBlockSize;
+ dctx->tmpInSize = 0;
+ dctx->tmpInTarget = 0;
+ dctx->tmpOut = dctx->tmpOutBuffer;
+ dctx->tmpOutStart = 0;
+ dctx->tmpOutSize = 0;
+
+ dctx->dStage = dstage_getBlockHeader;
/* fall-through */
- case dstage_getCBlockSize:
+ case dstage_getBlockHeader:
if ((size_t)(srcEnd - srcPtr) >= BHSize) {
selectedIn = srcPtr;
srcPtr += BHSize;
} else {
/* not enough input to read cBlockSize field */
- dctxPtr->tmpInSize = 0;
- dctxPtr->dStage = dstage_storeCBlockSize;
+ dctx->tmpInSize = 0;
+ dctx->dStage = dstage_storeBlockHeader;
}
- if (dctxPtr->dStage == dstage_storeCBlockSize) /* can be skipped */
- case dstage_storeCBlockSize:
- { size_t sizeToCopy = BHSize - dctxPtr->tmpInSize;
+ if (dctx->dStage == dstage_storeBlockHeader) /* can be skipped */
+ case dstage_storeBlockHeader:
+ { size_t sizeToCopy = BHSize - dctx->tmpInSize;
if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr;
- memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
+ memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy);
srcPtr += sizeToCopy;
- dctxPtr->tmpInSize += sizeToCopy;
- if (dctxPtr->tmpInSize < BHSize) { /* not enough input for cBlockSize */
- nextSrcSizeHint = BHSize - dctxPtr->tmpInSize;
+ dctx->tmpInSize += sizeToCopy;
+ if (dctx->tmpInSize < BHSize) { /* not enough input for cBlockSize */
+ nextSrcSizeHint = BHSize - dctx->tmpInSize;
doAnotherStage = 0;
break;
}
- selectedIn = dctxPtr->tmpIn;
+ selectedIn = dctx->tmpIn;
}
- /* case dstage_decodeCBlockSize: */ /* no more direct access, to remove scan-build warning */
+ /* decode block header */
{ size_t const nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU;
- if (nextCBlockSize==0) { /* frameEnd signal, no more CBlock */
- dctxPtr->dStage = dstage_getSuffix;
+ size_t const crcSize = dctx->frameInfo.blockChecksumFlag * 4;
+ if (nextCBlockSize==0) { /* frameEnd signal, no more block */
+ dctx->dStage = dstage_getSuffix;
break;
}
- if (nextCBlockSize > dctxPtr->maxBlockSize)
+ if (nextCBlockSize > dctx->maxBlockSize)
return err0r(LZ4F_ERROR_maxBlockSize_invalid);
- dctxPtr->tmpInTarget = nextCBlockSize;
if (LZ4F_readLE32(selectedIn) & LZ4F_BLOCKUNCOMPRESSED_FLAG) {
- dctxPtr->dStage = dstage_copyDirect;
+ /* next block is uncompressed */
+ dctx->tmpInTarget = nextCBlockSize;
+ if (dctx->frameInfo.blockChecksumFlag) {
+ XXH32_reset(&dctx->blockChecksum, 0);
+ }
+ dctx->dStage = dstage_copyDirect;
break;
}
- dctxPtr->dStage = dstage_getCBlock;
+ /* next block is a compressed block */
+ dctx->tmpInTarget = nextCBlockSize + crcSize;
+ dctx->dStage = dstage_getCBlock;
if (dstPtr==dstEnd) {
- nextSrcSizeHint = nextCBlockSize + BHSize;
+ nextSrcSizeHint = nextCBlockSize + crcSize + BHSize;
doAnotherStage = 0;
}
break;
}
case dstage_copyDirect: /* uncompressed block */
- { size_t sizeToCopy = dctxPtr->tmpInTarget;
- if ((size_t)(srcEnd-srcPtr) < sizeToCopy) sizeToCopy = srcEnd - srcPtr;
- if ((size_t)(dstEnd-dstPtr) < sizeToCopy) sizeToCopy = dstEnd - dstPtr;
+ { size_t const minBuffSize = MIN((size_t)(srcEnd-srcPtr), (size_t)(dstEnd-dstPtr));
+ size_t const sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize);
memcpy(dstPtr, srcPtr, sizeToCopy);
- if (dctxPtr->frameInfo.contentChecksumFlag)
- XXH32_update(&(dctxPtr->xxh), srcPtr, sizeToCopy);
- if (dctxPtr->frameInfo.contentSize)
- dctxPtr->frameRemainingSize -= sizeToCopy;
+ if (dctx->frameInfo.blockChecksumFlag) {
+ XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy);
+ }
+ if (dctx->frameInfo.contentChecksumFlag)
+ XXH32_update(&dctx->xxh, srcPtr, sizeToCopy);
+ if (dctx->frameInfo.contentSize)
+ dctx->frameRemainingSize -= sizeToCopy;
- /* dictionary management */
- if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked)
- LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 0);
+ /* history management (linked blocks only)*/
+ if (dctx->frameInfo.blockMode == LZ4F_blockLinked)
+ LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 0);
srcPtr += sizeToCopy;
dstPtr += sizeToCopy;
- if (sizeToCopy == dctxPtr->tmpInTarget) { /* all copied */
- dctxPtr->dStage = dstage_getCBlockSize;
+ if (sizeToCopy == dctx->tmpInTarget) { /* all done */
+ if (dctx->frameInfo.blockChecksumFlag) {
+ dctx->tmpInSize = 0;
+ dctx->dStage = dstage_getBlockChecksum;
+ } else
+ dctx->dStage = dstage_getBlockHeader; /* new block */
break;
}
- dctxPtr->tmpInTarget -= sizeToCopy; /* still need to copy more */
- nextSrcSizeHint = dctxPtr->tmpInTarget + BHSize;
+ dctx->tmpInTarget -= sizeToCopy; /* need to copy more */
+ nextSrcSizeHint = dctx->tmpInTarget +
+ + dctx->frameInfo.contentChecksumFlag * 4 /* block checksum */
+ + BHSize /* next header size */;
doAnotherStage = 0;
break;
}
+ /* check block checksum for recently transferred uncompressed block */
+ case dstage_getBlockChecksum:
+ { const void* crcSrc;
+ if ((srcEnd-srcPtr >= 4) && (dctx->tmpInSize==0)) {
+ crcSrc = srcPtr;
+ srcPtr += 4;
+ } else {
+ size_t const stillToCopy = 4 - dctx->tmpInSize;
+ size_t const sizeToCopy = MIN(stillToCopy, (size_t)(srcEnd-srcPtr));
+ memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);
+ dctx->tmpInSize += sizeToCopy;
+ srcPtr += sizeToCopy;
+ if (dctx->tmpInSize < 4) { /* all input consumed */
+ doAnotherStage = 0;
+ break;
+ }
+ crcSrc = dctx->header;
+ }
+ { U32 const readCRC = LZ4F_readLE32(crcSrc);
+ U32 const calcCRC = XXH32_digest(&dctx->blockChecksum);
+ if (readCRC != calcCRC)
+ return err0r(LZ4F_ERROR_blockChecksum_invalid);
+ }
+ }
+ dctx->dStage = dstage_getBlockHeader; /* new block */
+ break;
+
case dstage_getCBlock: /* entry from dstage_decodeCBlockSize */
- if ((size_t)(srcEnd-srcPtr) < dctxPtr->tmpInTarget) {
- dctxPtr->tmpInSize = 0;
- dctxPtr->dStage = dstage_storeCBlock;
+ if ((size_t)(srcEnd-srcPtr) < dctx->tmpInTarget) {
+ dctx->tmpInSize = 0;
+ dctx->dStage = dstage_storeCBlock;
break;
}
+ /* input large enough to read full block directly */
selectedIn = srcPtr;
- srcPtr += dctxPtr->tmpInTarget;
- dctxPtr->dStage = dstage_decodeCBlock;
+ srcPtr += dctx->tmpInTarget;
+ dctx->dStage = dstage_decodeCBlock;
break;
case dstage_storeCBlock:
- { size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize;
- if (sizeToCopy > (size_t)(srcEnd-srcPtr)) sizeToCopy = srcEnd-srcPtr;
- memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
- dctxPtr->tmpInSize += sizeToCopy;
+ { size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, (size_t)(srcEnd-srcPtr));
+ memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy);
+ dctx->tmpInSize += sizeToCopy;
srcPtr += sizeToCopy;
- if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) { /* need more input */
- nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + BHSize;
+ if (dctx->tmpInSize < dctx->tmpInTarget) { /* need more input */
+ nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize;
doAnotherStage=0;
break;
}
- selectedIn = dctxPtr->tmpIn;
- dctxPtr->dStage = dstage_decodeCBlock;
+ selectedIn = dctx->tmpIn;
+ dctx->dStage = dstage_decodeCBlock;
}
/* fall-through */
+ /* At this stage, input is large enough to decode a block */
case dstage_decodeCBlock:
- if ((size_t)(dstEnd-dstPtr) < dctxPtr->maxBlockSize) /* not enough place into dst : decode into tmpOut */
- dctxPtr->dStage = dstage_decodeCBlock_intoTmp;
+ if (dctx->frameInfo.blockChecksumFlag) {
+ dctx->tmpInTarget -= 4;
+ { U32 const readBlockCrc = LZ4F_readLE32(selectedIn + dctx->tmpInTarget);
+ U32 const calcBlockCrc = XXH32(selectedIn, dctx->tmpInTarget, 0);
+ if (readBlockCrc != calcBlockCrc)
+ return err0r(LZ4F_ERROR_blockChecksum_invalid);
+ } }
+ if ((size_t)(dstEnd-dstPtr) < dctx->maxBlockSize) /* not enough place into dst : decode into tmpOut */
+ dctx->dStage = dstage_decodeCBlock_intoTmp;
else
- dctxPtr->dStage = dstage_decodeCBlock_intoDst;
+ dctx->dStage = dstage_decodeCBlock_intoDst;
break;
case dstage_decodeCBlock_intoDst:
- { int (*decoder)(const char*, char*, int, int, const char*, int);
- int decodedSize;
-
- if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked)
- decoder = LZ4_decompress_safe_usingDict;
- else
- decoder = LZ4F_decompress_safe;
-
- decodedSize = decoder((const char*)selectedIn, (char*)dstPtr,
- (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize,
- (const char*)dctxPtr->dict, (int)dctxPtr->dictSize);
+ { int const decodedSize = LZ4_decompress_safe_usingDict(
+ (const char*)selectedIn, (char*)dstPtr,
+ (int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
+ (const char*)dctx->dict, (int)dctx->dictSize);
if (decodedSize < 0) return err0r(LZ4F_ERROR_GENERIC); /* decompression failed */
- if (dctxPtr->frameInfo.contentChecksumFlag)
- XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize);
- if (dctxPtr->frameInfo.contentSize)
- dctxPtr->frameRemainingSize -= decodedSize;
+ if (dctx->frameInfo.contentChecksumFlag)
+ XXH32_update(&(dctx->xxh), dstPtr, decodedSize);
+ if (dctx->frameInfo.contentSize)
+ dctx->frameRemainingSize -= decodedSize;
/* dictionary management */
- if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked)
- LZ4F_updateDict(dctxPtr, dstPtr, decodedSize, dstStart, 0);
+ if (dctx->frameInfo.blockMode==LZ4F_blockLinked)
+ LZ4F_updateDict(dctx, dstPtr, decodedSize, dstStart, 0);
dstPtr += decodedSize;
- dctxPtr->dStage = dstage_getCBlockSize;
+ dctx->dStage = dstage_getBlockHeader;
break;
}
case dstage_decodeCBlock_intoTmp:
/* not enough place into dst : decode into tmpOut */
- { int (*decoder)(const char*, char*, int, int, const char*, int);
- int decodedSize;
-
- if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked)
- decoder = LZ4_decompress_safe_usingDict;
- else
- decoder = LZ4F_decompress_safe;
-
- /* ensure enough place for tmpOut */
- if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked) {
- if (dctxPtr->dict == dctxPtr->tmpOutBuffer) {
- if (dctxPtr->dictSize > 128 KB) {
- memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - 64 KB, 64 KB);
- dctxPtr->dictSize = 64 KB;
- }
- dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + dctxPtr->dictSize;
- } else { /* dict not within tmp */
- size_t reservedDictSpace = dctxPtr->dictSize;
- if (reservedDictSpace > 64 KB) reservedDictSpace = 64 KB;
- dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + reservedDictSpace;
+
+ /* ensure enough place for tmpOut */
+ if (dctx->frameInfo.blockMode == LZ4F_blockLinked) {
+ if (dctx->dict == dctx->tmpOutBuffer) {
+ if (dctx->dictSize > 128 KB) {
+ memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - 64 KB, 64 KB);
+ dctx->dictSize = 64 KB;
}
+ dctx->tmpOut = dctx->tmpOutBuffer + dctx->dictSize;
+ } else { /* dict not within tmp */
+ size_t const reservedDictSpace = MIN(dctx->dictSize, 64 KB);
+ dctx->tmpOut = dctx->tmpOutBuffer + reservedDictSpace;
}
+ }
- /* Decode */
- decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut,
- (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize,
- (const char*)dctxPtr->dict, (int)dctxPtr->dictSize);
- if (decodedSize < 0)
- return err0r(LZ4F_ERROR_decompressionFailed); /* decompression failed */
- if (dctxPtr->frameInfo.contentChecksumFlag)
- XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize);
- if (dctxPtr->frameInfo.contentSize)
- dctxPtr->frameRemainingSize -= decodedSize;
- dctxPtr->tmpOutSize = decodedSize;
- dctxPtr->tmpOutStart = 0;
- dctxPtr->dStage = dstage_flushOut;
- break;
+ /* Decode block */
+ { int const decodedSize = LZ4_decompress_safe_usingDict(
+ (const char*)selectedIn, (char*)dctx->tmpOut,
+ (int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
+ (const char*)dctx->dict, (int)dctx->dictSize);
+ if (decodedSize < 0) /* decompression failed */
+ return err0r(LZ4F_ERROR_decompressionFailed);
+ if (dctx->frameInfo.contentChecksumFlag)
+ XXH32_update(&(dctx->xxh), dctx->tmpOut, decodedSize);
+ if (dctx->frameInfo.contentSize)
+ dctx->frameRemainingSize -= decodedSize;
+ dctx->tmpOutSize = decodedSize;
+ dctx->tmpOutStart = 0;
+ dctx->dStage = dstage_flushOut;
}
+ /* fall-through */
case dstage_flushOut: /* flush decoded data from tmpOut to dstBuffer */
- { size_t sizeToCopy = dctxPtr->tmpOutSize - dctxPtr->tmpOutStart;
- if (sizeToCopy > (size_t)(dstEnd-dstPtr)) sizeToCopy = dstEnd-dstPtr;
- memcpy(dstPtr, dctxPtr->tmpOut + dctxPtr->tmpOutStart, sizeToCopy);
+ { size_t const sizeToCopy = MIN(dctx->tmpOutSize - dctx->tmpOutStart, (size_t)(dstEnd-dstPtr));
+ memcpy(dstPtr, dctx->tmpOut + dctx->tmpOutStart, sizeToCopy);
/* dictionary management */
- if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked)
- LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 1);
+ if (dctx->frameInfo.blockMode==LZ4F_blockLinked)
+ LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 1);
- dctxPtr->tmpOutStart += sizeToCopy;
+ dctx->tmpOutStart += sizeToCopy;
dstPtr += sizeToCopy;
- /* end of flush ? */
- if (dctxPtr->tmpOutStart == dctxPtr->tmpOutSize) {
- dctxPtr->dStage = dstage_getCBlockSize;
+ if (dctx->tmpOutStart == dctx->tmpOutSize) { /* all flushed */
+ dctx->dStage = dstage_getBlockHeader; /* get next block */
break;
}
nextSrcSizeHint = BHSize;
@@ -1337,46 +1518,47 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
}
case dstage_getSuffix:
- { size_t const suffixSize = dctxPtr->frameInfo.contentChecksumFlag * 4;
- if (dctxPtr->frameRemainingSize)
+ { size_t const suffixSize = dctx->frameInfo.contentChecksumFlag * 4;
+ if (dctx->frameRemainingSize)
return err0r(LZ4F_ERROR_frameSize_wrong); /* incorrect frame size decoded */
if (suffixSize == 0) { /* frame completed */
nextSrcSizeHint = 0;
- dctxPtr->dStage = dstage_getHeader;
+ LZ4F_resetDecompressionContext(dctx);
doAnotherStage = 0;
break;
}
if ((srcEnd - srcPtr) < 4) { /* not enough size for entire CRC */
- dctxPtr->tmpInSize = 0;
- dctxPtr->dStage = dstage_storeSuffix;
+ dctx->tmpInSize = 0;
+ dctx->dStage = dstage_storeSuffix;
} else {
selectedIn = srcPtr;
srcPtr += 4;
}
}
- if (dctxPtr->dStage == dstage_storeSuffix) /* can be skipped */
+ if (dctx->dStage == dstage_storeSuffix) /* can be skipped */
case dstage_storeSuffix:
{
- size_t sizeToCopy = 4 - dctxPtr->tmpInSize;
+ size_t sizeToCopy = 4 - dctx->tmpInSize;
if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr;
- memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
+ memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy);
srcPtr += sizeToCopy;
- dctxPtr->tmpInSize += sizeToCopy;
- if (dctxPtr->tmpInSize < 4) { /* not enough input to read complete suffix */
- nextSrcSizeHint = 4 - dctxPtr->tmpInSize;
+ dctx->tmpInSize += sizeToCopy;
+ if (dctx->tmpInSize < 4) { /* not enough input to read complete suffix */
+ nextSrcSizeHint = 4 - dctx->tmpInSize;
doAnotherStage=0;
break;
}
- selectedIn = dctxPtr->tmpIn;
+ selectedIn = dctx->tmpIn;
}
- /* case dstage_checkSuffix: */ /* no direct call, to avoid scan-build warning */
+ /* case dstage_checkSuffix: */ /* no direct call, avoid scan-build warning */
{ U32 const readCRC = LZ4F_readLE32(selectedIn);
- U32 const resultCRC = XXH32_digest(&(dctxPtr->xxh));
- if (readCRC != resultCRC) return err0r(LZ4F_ERROR_contentChecksum_invalid);
+ U32 const resultCRC = XXH32_digest(&(dctx->xxh));
+ if (readCRC != resultCRC)
+ return err0r(LZ4F_ERROR_contentChecksum_invalid);
nextSrcSizeHint = 0;
- dctxPtr->dStage = dstage_getHeader;
+ LZ4F_resetDecompressionContext(dctx);
doAnotherStage = 0;
break;
}
@@ -1387,78 +1569,77 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
srcPtr += 4;
} else {
/* not enough input to read cBlockSize field */
- dctxPtr->tmpInSize = 4;
- dctxPtr->tmpInTarget = 8;
- dctxPtr->dStage = dstage_storeSFrameSize;
+ dctx->tmpInSize = 4;
+ dctx->tmpInTarget = 8;
+ dctx->dStage = dstage_storeSFrameSize;
}
- if (dctxPtr->dStage == dstage_storeSFrameSize)
+ if (dctx->dStage == dstage_storeSFrameSize)
case dstage_storeSFrameSize:
{
- size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize;
- if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr;
- memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
+ size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize,
+ (size_t)(srcEnd - srcPtr) );
+ memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);
srcPtr += sizeToCopy;
- dctxPtr->tmpInSize += sizeToCopy;
- if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) {
+ dctx->tmpInSize += sizeToCopy;
+ if (dctx->tmpInSize < dctx->tmpInTarget) {
/* not enough input to get full sBlockSize; wait for more */
- nextSrcSizeHint = dctxPtr->tmpInTarget - dctxPtr->tmpInSize;
+ nextSrcSizeHint = dctx->tmpInTarget - dctx->tmpInSize;
doAnotherStage = 0;
break;
}
- selectedIn = dctxPtr->header + 4;
+ selectedIn = dctx->header + 4;
}
/* case dstage_decodeSFrameSize: */ /* no direct access */
{ size_t const SFrameSize = LZ4F_readLE32(selectedIn);
- dctxPtr->frameInfo.contentSize = SFrameSize;
- dctxPtr->tmpInTarget = SFrameSize;
- dctxPtr->dStage = dstage_skipSkippable;
+ dctx->frameInfo.contentSize = SFrameSize;
+ dctx->tmpInTarget = SFrameSize;
+ dctx->dStage = dstage_skipSkippable;
break;
}
case dstage_skipSkippable:
- { size_t skipSize = dctxPtr->tmpInTarget;
- if (skipSize > (size_t)(srcEnd-srcPtr)) skipSize = srcEnd-srcPtr;
+ { size_t const skipSize = MIN(dctx->tmpInTarget, (size_t)(srcEnd-srcPtr));
srcPtr += skipSize;
- dctxPtr->tmpInTarget -= skipSize;
+ dctx->tmpInTarget -= skipSize;
doAnotherStage = 0;
- nextSrcSizeHint = dctxPtr->tmpInTarget;
- if (nextSrcSizeHint) break;
- dctxPtr->dStage = dstage_getHeader;
+ nextSrcSizeHint = dctx->tmpInTarget;
+ if (nextSrcSizeHint) break; /* still more to skip */
+ LZ4F_resetDecompressionContext(dctx);
break;
}
}
}
- /* preserve dictionary within tmp if necessary */
- if ( (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked)
- &&(dctxPtr->dict != dctxPtr->tmpOutBuffer)
- &&(!decompressOptionsPtr->stableDst)
- &&((unsigned)(dctxPtr->dStage-1) < (unsigned)(dstage_getSuffix-1))
- )
+ /* preserve history within tmp if necessary */
+ if ( (dctx->frameInfo.blockMode==LZ4F_blockLinked)
+ && (dctx->dict != dctx->tmpOutBuffer)
+ && (dctx->dStage != dstage_getFrameHeader)
+ && (!decompressOptionsPtr->stableDst)
+ && ((unsigned)(dctx->dStage-1) < (unsigned)(dstage_getSuffix-1)) )
{
- if (dctxPtr->dStage == dstage_flushOut) {
- size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer;
- size_t copySize = 64 KB - dctxPtr->tmpOutSize;
- const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart;
- if (dctxPtr->tmpOutSize > 64 KB) copySize = 0;
+ if (dctx->dStage == dstage_flushOut) {
+ size_t preserveSize = dctx->tmpOut - dctx->tmpOutBuffer;
+ size_t copySize = 64 KB - dctx->tmpOutSize;
+ const BYTE* oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart;
+ if (dctx->tmpOutSize > 64 KB) copySize = 0;
if (copySize > preserveSize) copySize = preserveSize;
- memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize);
+ memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize);
- dctxPtr->dict = dctxPtr->tmpOutBuffer;
- dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart;
+ dctx->dict = dctx->tmpOutBuffer;
+ dctx->dictSize = preserveSize + dctx->tmpOutStart;
} else {
- size_t newDictSize = dctxPtr->dictSize;
- const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize;
+ size_t newDictSize = dctx->dictSize;
+ const BYTE* oldDictEnd = dctx->dict + dctx->dictSize;
if ((newDictSize) > 64 KB) newDictSize = 64 KB;
- memcpy(dctxPtr->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize);
+ memcpy(dctx->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize);
- dctxPtr->dict = dctxPtr->tmpOutBuffer;
- dctxPtr->dictSize = newDictSize;
- dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + newDictSize;
+ dctx->dict = dctx->tmpOutBuffer;
+ dctx->dictSize = newDictSize;
+ dctx->tmpOut = dctx->tmpOutBuffer + newDictSize;
}
}
@@ -1466,3 +1647,23 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
*dstSizePtr = (dstPtr - dstStart);
return nextSrcSizeHint;
}
+
+/*! LZ4F_decompress_usingDict() :
+ * Same as LZ4F_decompress(), using a predefined dictionary.
+ * Dictionary is used "in place", without any preprocessing.
+ * It must remain accessible throughout the entire frame decoding.
+ */
+size_t LZ4F_decompress_usingDict(LZ4F_dctx* dctx,
+ void* dstBuffer, size_t* dstSizePtr,
+ const void* srcBuffer, size_t* srcSizePtr,
+ const void* dict, size_t dictSize,
+ const LZ4F_decompressOptions_t* decompressOptionsPtr)
+{
+ if (dctx->dStage <= dstage_init) {
+ dctx->dict = (const BYTE*)dict;
+ dctx->dictSize = dictSize;
+ }
+ return LZ4F_decompress(dctx, dstBuffer, dstSizePtr,
+ srcBuffer, srcSizePtr,
+ decompressOptionsPtr);
+}
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index dd2be58..88a6513 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -100,8 +100,8 @@ LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return
/*-************************************
* Frame compression types
**************************************/
-/* #define LZ4F_DISABLE_OBSOLETE_ENUMS */ /* uncomment to disable obsolete enums */
-#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS
+/* #define LZ4F_ENABLE_OBSOLETE_ENUMS // uncomment to enable obsolete enums */
+#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS
# define LZ4F_OBSOLETE_ENUM(x) , LZ4F_DEPRECATE(x) = LZ4F_##x
#else
# define LZ4F_OBSOLETE_ENUM(x)
@@ -140,12 +140,17 @@ typedef enum {
} LZ4F_contentChecksum_t;
typedef enum {
+ LZ4F_noBlockChecksum=0,
+ LZ4F_blockChecksumEnabled
+} LZ4F_blockChecksum_t;
+
+typedef enum {
LZ4F_frame=0,
LZ4F_skippableFrame
LZ4F_OBSOLETE_ENUM(skippableFrame)
} LZ4F_frameType_t;
-#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS
+#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS
typedef LZ4F_blockSizeID_t blockSizeID_t;
typedef LZ4F_blockMode_t blockMode_t;
typedef LZ4F_frameType_t frameType_t;
@@ -153,50 +158,52 @@ typedef LZ4F_contentChecksum_t contentChecksum_t;
#endif
/*! LZ4F_frameInfo_t :
- * makes it possible to supply detailed frame parameters to the stream interface.
- * It's not required to set all fields, as long as the structure was initially memset() to zero.
- * All reserved fields must be set to zero. */
+ * makes it possible to set or read frame parameters.
+ * It's not required to set all fields, as long as the structure was initially memset() to zero.
+ * For all fields, 0 sets it to default value */
typedef struct {
- LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
- LZ4F_blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */
- LZ4F_contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */
- LZ4F_frameType_t frameType; /* LZ4F_frame, skippableFrame ; 0 == default */
- unsigned long long contentSize; /* Size of uncompressed (original) content ; 0 == unknown */
- unsigned reserved[2]; /* must be zero for forward compatibility */
+ LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
+ LZ4F_blockMode_t blockMode; /* LZ4F_blockLinked, LZ4F_blockIndependent ; 0 == default */
+ LZ4F_contentChecksum_t contentChecksumFlag; /* if enabled, frame is terminated with a 32-bits checksum of decompressed data ; 0 == disabled (default) */
+ LZ4F_frameType_t frameType; /* read-only field : LZ4F_frame or LZ4F_skippableFrame */
+ unsigned long long contentSize; /* Size of uncompressed content ; 0 == unknown */
+ unsigned dictID; /* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */
+ LZ4F_blockChecksum_t blockChecksumFlag; /* if enabled, each block is followed by a checksum of block's compressed data ; 0 == disabled (default) */
} LZ4F_frameInfo_t;
/*! LZ4F_preferences_t :
- * makes it possible to supply detailed compression parameters to the stream interface.
- * It's not required to set all fields, as long as the structure was initially memset() to zero.
- * All reserved fields must be set to zero. */
+ * makes it possible to supply detailed compression parameters to the stream interface.
+ * It's not required to set all fields, as long as the structure was initially memset() to zero.
+ * All reserved fields must be set to zero. */
typedef struct {
LZ4F_frameInfo_t frameInfo;
int compressionLevel; /* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */
- unsigned autoFlush; /* 1 == always flush (reduce usage of tmp buffer) */
+ unsigned autoFlush; /* 1 == always flush, to reduce usage of internal buffers */
unsigned reserved[4]; /* must be zero for forward compatibility */
} LZ4F_preferences_t;
+LZ4FLIB_API int LZ4F_compressionLevel_max(void);
+
/*-*********************************
* Simple compression function
***********************************/
-/*!LZ4F_compressFrameBound() :
- * Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
- * Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
+/*! LZ4F_compressFrameBound() :
+ * Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
+ * Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
*/
LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
-/*!LZ4F_compressFrame() :
- * Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.1
- * An important rule is that dstBuffer MUST be large enough (dstCapacity) to store the result in worst case situation.
- * This value is supplied by LZ4F_compressFrameBound().
- * If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode).
- * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
+/*! LZ4F_compressFrame() :
+ * Compress an entire srcBuffer into a valid LZ4 frame.
+ * dstCapacity MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
+ * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
* @return : number of bytes written into dstBuffer.
* or an error code if it fails (can be tested using LZ4F_isError())
*/
-LZ4FLIB_API size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
-
+LZ4FLIB_API size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
+ const void* srcBuffer, size_t srcSize,
+ const LZ4F_preferences_t* preferencesPtr);
/*-***********************************
@@ -228,15 +235,17 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
/*---- Compression ----*/
-#define LZ4F_HEADER_SIZE_MAX 15
+#define LZ4F_HEADER_SIZE_MAX 19
/*! LZ4F_compressBegin() :
- * will write the frame header into dstBuffer.
- * dstCapacity must be large enough to store the header. Maximum header size is LZ4F_HEADER_SIZE_MAX bytes.
+ * will write the frame header into dstBuffer.
+ * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
* `prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default.
* @return : number of bytes written into dstBuffer for the header
* or an error code (which can be tested using LZ4F_isError())
*/
-LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* prefsPtr);
+LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx,
+ void* dstBuffer, size_t dstCapacity,
+ const LZ4F_preferences_t* prefsPtr);
/*! LZ4F_compressBound() :
* Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
@@ -269,13 +278,13 @@ LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t
LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
/*! LZ4F_compressEnd() :
- * To properly finish an LZ4 frame, invoke LZ4F_compressEnd().
- * It will flush whatever data remained within `cctx` (like LZ4_flush())
- * and properly finalize the frame, with an endMark and a checksum.
+ * To properly finish an LZ4 frame, invoke LZ4F_compressEnd().
+ * It will flush whatever data remained within `cctx` (like LZ4_flush())
+ * and properly finalize the frame, with an endMark and a checksum.
* `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default.
* @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled)
* or an error code if it fails (which can be tested using LZ4F_isError())
- * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.
+ * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.
*/
LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
@@ -288,7 +297,7 @@ typedef LZ4F_dctx* LZ4F_decompressionContext_t; /* compatibility with previous
typedef struct {
unsigned stableDst; /* pledge that at least 64KB+64Bytes of previously decompressed data remain unmodifed where it was decoded. This optimization skips storage operations in tmp buffers */
- unsigned reserved[3];
+ unsigned reserved[3]; /* must be set to zero for forward compatibility */
} LZ4F_decompressOptions_t;
@@ -312,10 +321,10 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
*************************************/
/*! LZ4F_getFrameInfo() :
- * This function extracts frame parameters (such as max blockSize, frame checksum, etc.).
- * Its usage is optional.
- * Extracted information can typically be useful for allocation purposes.
- * This function works in 2 situations :
+ * This function extracts frame parameters (max blockSize, dictID, etc.).
+ * Its usage is optional.
+ * Extracted information is typically useful for allocation and dictionary.
+ * This function works in 2 situations :
* - At the beginning of a new frame, in which case
* it will decode information from `srcBuffer`, starting the decoding process.
* Input size must be large enough to successfully decode the entire frame header.
@@ -323,41 +332,42 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
* It's allowed to provide more input data than this minimum.
* - After decoding has been started.
* In which case, no input is read, frame parameters are extracted from dctx.
- * - If decoding has barely started, but not yet extracted information from header, LZ4F_getFrameInfo() will fail.
- * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
- * Decompression must resume from (srcBuffer + *srcSizePtr).
+ * - If decoding has barely started, but not yet extracted information from header,
+ * LZ4F_getFrameInfo() will fail.
+ * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
+ * Decompression must resume from (srcBuffer + *srcSizePtr).
* @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
- * or an error code which can be tested using LZ4F_isError()
- * note 1 : in case of error, dctx is not modified. Decoding operation can resume safely.
- * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
+ * or an error code which can be tested using LZ4F_isError().
+ * note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely.
+ * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
*/
LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
LZ4F_frameInfo_t* frameInfoPtr,
const void* srcBuffer, size_t* srcSizePtr);
/*! LZ4F_decompress() :
- * Call this function repetitively to regenerate compressed data from `srcBuffer`.
- * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
+ * Call this function repetitively to regenerate compressed data from `srcBuffer`.
+ * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
*
- * The number of bytes regenerated into dstBuffer is provided within *dstSizePtr (necessarily <= original value).
+ * The number of bytes regenerated into dstBuffer is provided within *dstSizePtr (necessarily <= original value).
*
- * The number of bytes consumed from srcBuffer is provided within *srcSizePtr (necessarily <= original value).
- * Number of bytes consumed can be < number of bytes provided.
- * It typically happens when dstBuffer is not large enough to contain all decoded data.
- * Unconsumed source data must be presented again in subsequent invocations.
+ * The number of bytes consumed from srcBuffer is provided within *srcSizePtr (necessarily <= original value).
+ * Number of bytes consumed can be < number of bytes provided.
+ * It typically happens when dstBuffer is not large enough to contain all decoded data.
+ * Unconsumed source data must be presented again in subsequent invocations.
*
* `dstBuffer` content is expected to be flushed between each invocation, as its content will be overwritten.
* `dstBuffer` itself can be changed at will between each consecutive function invocation.
*
- * @return is an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
- * Schematically, it's the size of the current (or remaining) compressed block + header of next block.
- * Respecting the hint provides some small speed benefit, because it skips intermediate buffers.
- * This is just a hint though, it's always possible to provide any srcSize.
- * When a frame is fully decoded, @return will be 0 (no more data expected).
- * If decompression failed, @return is an error code, which can be tested using LZ4F_isError().
+ * @return : an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
+ * Schematically, it's the size of the current (or remaining) compressed block + header of next block.
+ * Respecting the hint provides some small speed benefit, because it skips intermediate buffers.
+ * This is just a hint though, it's always possible to provide any srcSize.
+ * When a frame is fully decoded, @return will be 0 (no more data expected).
+ * If decompression failed, @return is an error code, which can be tested using LZ4F_isError().
*
- * After a frame is fully decoded, dctx can be used again to decompress another frame.
- * After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state.
+ * After a frame is fully decoded, dctx can be used again to decompress another frame.
+ * After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state.
*/
LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx,
void* dstBuffer, size_t* dstSizePtr,
diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h
index d3bae82..1899f8e 100644
--- a/lib/lz4frame_static.h
+++ b/lib/lz4frame_static.h
@@ -59,7 +59,7 @@ extern "C" {
ITEM(ERROR_contentChecksumFlag_invalid) \
ITEM(ERROR_compressionLevel_invalid) \
ITEM(ERROR_headerVersion_wrong) \
- ITEM(ERROR_blockChecksum_unsupported) \
+ ITEM(ERROR_blockChecksum_invalid) \
ITEM(ERROR_reservedFlag_set) \
ITEM(ERROR_allocation_failed) \
ITEM(ERROR_srcSize_tooLarge) \
@@ -82,6 +82,60 @@ typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes;
LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult);
+
+/**********************************
+ * Bulk processing dictionary API
+ *********************************/
+typedef struct LZ4F_CDict_s LZ4F_CDict;
+
+/*! LZ4_createCDict() :
+ * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
+ * LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+ * LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
+ * `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict */
+LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize);
+void LZ4F_freeCDict(LZ4F_CDict* CDict);
+
+
+/*! LZ4_compressFrame_usingCDict() :
+ * Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary.
+ * If cdict==NULL, compress without a dictionary.
+ * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
+ * If this condition is not respected, function will fail (@return an errorCode).
+ * The LZ4F_preferences_t structure is optional : you may provide NULL as argument,
+ * but it's not recommended, as it's the only way to provide dictID in the frame header.
+ * @return : number of bytes written into dstBuffer.
+ * or an error code if it fails (can be tested using LZ4F_isError()) */
+size_t LZ4F_compressFrame_usingCDict(void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const LZ4F_CDict* cdict,
+ const LZ4F_preferences_t* preferencesPtr);
+
+
+/*! LZ4F_compressBegin_usingCDict() :
+ * Inits streaming dictionary compression, and writes the frame header into dstBuffer.
+ * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
+ * `prefsPtr` is optional : you may provide NULL as argument,
+ * however, it's the only way to provide dictID in the frame header.
+ * @return : number of bytes written into dstBuffer for the header,
+ * or an error code (which can be tested using LZ4F_isError()) */
+size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctx,
+ void* dstBuffer, size_t dstCapacity,
+ const LZ4F_CDict* cdict,
+ const LZ4F_preferences_t* prefsPtr);
+
+
+/*! LZ4F_decompress_usingDict() :
+ * Same as LZ4F_decompress(), using a predefined dictionary.
+ * Dictionary is used "in place", without any preprocessing.
+ * It must remain accessible throughout the entire frame decoding. */
+size_t LZ4F_decompress_usingDict(LZ4F_dctx* dctxPtr,
+ void* dstBuffer, size_t* dstSizePtr,
+ const void* srcBuffer, size_t* srcSizePtr,
+ const void* dict, size_t dictSize,
+ const LZ4F_decompressOptions_t* decompressOptionsPtr);
+
+
#if defined (__cplusplus)
}
#endif
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index ca9c2e6..22eb071 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -69,6 +69,8 @@
/*=== Macros ===*/
+#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
+#define MAX(a,b) ( (a) > (b) ? (a) : (b) )
#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG))
#define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */
#define DELTANEXTU16(table, pos) table[(U16)(pos)] /* faster */
@@ -610,7 +612,11 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i
**************************************/
/* allocation */
LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); }
-int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { free(LZ4_streamHCPtr); return 0; }
+int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) {
+ if (!LZ4_streamHCPtr) return 0; /* support free on NULL */
+ free(LZ4_streamHCPtr);
+ return 0;
+}
/* initialization */
@@ -623,6 +629,16 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
LZ4_streamHCPtr->internal_donotuse.searchNum = LZ4HC_getSearchNum(compressionLevel);
}
+void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
+{
+ int const currentCLevel = LZ4_streamHCPtr->internal_donotuse.compressionLevel;
+ int const minCLevel = currentCLevel < LZ4HC_CLEVEL_OPT_MIN ? 1 : LZ4HC_CLEVEL_OPT_MIN;
+ int const maxCLevel = currentCLevel < LZ4HC_CLEVEL_OPT_MIN ? LZ4HC_CLEVEL_OPT_MIN-1 : LZ4HC_CLEVEL_MAX;
+ compressionLevel = MIN(compressionLevel, minCLevel);
+ compressionLevel = MAX(compressionLevel, maxCLevel);
+ LZ4_streamHCPtr->internal_donotuse.compressionLevel = compressionLevel;
+}
+
int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize)
{
LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
@@ -767,7 +783,11 @@ void* LZ4_createHC (char* inputBuffer)
return hc4;
}
-int LZ4_freeHC (void* LZ4HC_Data) { FREEMEM(LZ4HC_Data); return 0; }
+int LZ4_freeHC (void* LZ4HC_Data) {
+ if (!LZ4HC_Data) return 0; /* support free on NULL */
+ FREEMEM(LZ4HC_Data);
+ return 0;
+}
int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int cLevel)
{
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index b63c825..66d5636 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -161,7 +161,7 @@ typedef struct
typedef struct
{
unsigned int hashTable[LZ4HC_HASHTABLESIZE];
- unsigned short chainTable[LZ4HC_MAXD];
+ unsigned short chainTable[LZ4HC_MAXD];
const unsigned char* end; /* next block here to continue on current prefix */
const unsigned char* base; /* All index relative to this position */
const unsigned char* dictBase; /* alternate base for extDict */
@@ -245,25 +245,34 @@ LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStr
* or 0 if compression fails.
* `srcSizePtr` : value will be updated to indicate how much bytes were read from `src`
*/
-LZ4LIB_API int LZ4_compress_HC_destSize(void* LZ4HC_Data,
+int LZ4_compress_HC_destSize(void* LZ4HC_Data,
const char* src, char* dst,
int* srcSizePtr, int targetDstSize,
int compressionLevel);
-/*! LZ4_compress_HC_continue_destSize() :
+/*! LZ4_compress_HC_continue_destSize() : v1.8.0 (experimental)
* Similar as LZ4_compress_HC_continue(),
* but will read a variable nb of bytes from `src`
* to fit into `targetDstSize` budget.
* Result is provided in 2 parts :
* @return : the number of bytes written into 'dst'
* or 0 if compression fails.
- * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src`
- * Important : due to limitations, this prototype only works well up to cLevel < LZ4HC_CLEVEL_OPT_MIN
- * beyond that level, compression performance will be much reduced due to internal incompatibilities
+ * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src`.
+ * Important : due to limitations, this prototype only works well up to cLevel < LZ4HC_CLEVEL_OPT_MIN
+ * beyond that level, compression performance will be much reduced due to internal incompatibilities
*/
-LZ4LIB_API int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr,
+int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr,
const char* src, char* dst,
int* srcSizePtr, int targetDstSize);
+/*! LZ4_setCompressionLevel() : v1.8.0 (experimental)
+ * It's possible to change compression level after LZ4_resetStreamHC(), between 2 invocations of LZ4_compress_HC_continue*(),
+ * but that requires to stay in the same mode (aka 1-10 or 11-12).
+ * This function ensures this condition.
+ */
+void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
+
+
+
#endif /* LZ4_HC_SLO_098092834 */
#endif /* LZ4_HC_STATIC_LINKING_ONLY */
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index 0cabc27..b4a3c14 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -138,7 +138,7 @@ static int usage_advanced(const char* exeName)
DISPLAY( " -l : compress using Legacy format (Linux kernel compression)\n");
DISPLAY( " -B# : Block size [4-7] (default : 7) \n");
DISPLAY( " -BD : Block dependency (improve compression ratio) \n");
- /* DISPLAY( " -BX : enable block checksum (default:disabled)\n"); *//* Option currently inactive */
+ DISPLAY( " -BX : enable block checksum (default:disabled) \n");
DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled) \n");
DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n");
DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n");
diff --git a/programs/lz4io.c b/programs/lz4io.c
index 1e6b437..06741b4 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -173,17 +173,17 @@ int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode)
return g_blockIndependence;
}
-/* Default setting : no checksum */
-int LZ4IO_setBlockChecksumMode(int xxhash)
+/* Default setting : no block checksum */
+int LZ4IO_setBlockChecksumMode(int enable)
{
- g_blockChecksum = (xxhash != 0);
+ g_blockChecksum = (enable != 0);
return g_blockChecksum;
}
/* Default setting : checksum enabled */
-int LZ4IO_setStreamChecksumMode(int xxhash)
+int LZ4IO_setStreamChecksumMode(int enable)
{
- g_streamChecksum = (xxhash != 0);
+ g_streamChecksum = (enable != 0);
return g_streamChecksum;
}
@@ -455,6 +455,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
prefs.compressionLevel = compressionLevel;
prefs.frameInfo.blockMode = (LZ4F_blockMode_t)g_blockIndependence;
prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)g_blockSizeId;
+ prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)g_blockChecksum;
prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)g_streamChecksum;
if (g_contentSizeFlag) {
U64 const fileSize = UTIL_getFileSize(srcFileName);
diff --git a/tests/Makefile b/tests/Makefile
index 5d2532f..f00778f 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -263,6 +263,7 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat
$(LZ4) -f tmp
cat tmp >> tmp.lz4
$(LZ4) -f tmp.lz4 # uncompress valid frame followed by invalid data
+ $(LZ4) -BX tmp -c -q | $(LZ4) -tv # test block checksum
@$(RM) tmp*
test-lz4-hugefile: lz4 datagen
diff --git a/tests/frametest.c b/tests/frametest.c
index 0dadf9f..88d0afd 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -164,6 +164,9 @@ static unsigned FUZ_highbit(U32 v32)
/*-*******************************************************
* Tests
*********************************************************/
+#define CHECK_V(v,f) v = f; if (LZ4F_isError(v)) goto _output_error
+#define CHECK(f) { LZ4F_errorCode_t const CHECK_V(err_ , f); }
+
int basicTests(U32 seed, double compressibility)
{
#define COMPRESSIBLE_NOISE_LENGTH (2 MB)
@@ -197,24 +200,20 @@ int basicTests(U32 seed, double compressibility)
/* Special case : null-content frame */
testSize = 0;
DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : ");
- cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL);
- if (LZ4F_isError(cSize)) goto _output_error;
+ CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL));
DISPLAYLEVEL(3, "null content encoded into a %u bytes frame \n", (unsigned)cSize);
DISPLAYLEVEL(3, "LZ4F_createDecompressionContext \n");
- { LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
- if (LZ4F_isError(errorCode)) goto _output_error; }
+ CHECK ( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) );
DISPLAYLEVEL(3, "LZ4F_getFrameInfo on null-content frame (#157) \n");
{ size_t avail_in = cSize;
LZ4F_frameInfo_t frame_info;
- LZ4F_errorCode_t const errorCode = LZ4F_getFrameInfo(dCtx, &frame_info, compressedBuffer, &avail_in);
- if (LZ4F_isError(errorCode)) goto _output_error;
+ CHECK( LZ4F_getFrameInfo(dCtx, &frame_info, compressedBuffer, &avail_in) );
}
DISPLAYLEVEL(3, "LZ4F_freeDecompressionContext \n");
- { LZ4F_errorCode_t const errorCode = LZ4F_freeDecompressionContext(dCtx);
- if (LZ4F_isError(errorCode)) goto _output_error; }
+ CHECK( LZ4F_freeDecompressionContext(dCtx) );
dCtx = NULL;
/* test one-pass frame compression */
@@ -224,26 +223,22 @@ int basicTests(U32 seed, double compressibility)
{ LZ4F_preferences_t fastCompressPrefs;
memset(&fastCompressPrefs, 0, sizeof(fastCompressPrefs));
fastCompressPrefs.compressionLevel = -3;
- cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, &fastCompressPrefs);
- if (LZ4F_isError(cSize)) goto _output_error;
+ CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, &fastCompressPrefs));
DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
}
DISPLAYLEVEL(3, "LZ4F_compressFrame, using default preferences : ");
- cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL);
- if (LZ4F_isError(cSize)) goto _output_error;
+ CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL));
DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
DISPLAYLEVEL(3, "Decompression test : \n");
{ size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
size_t compressedBufferSize = cSize;
- LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
- if (LZ4F_isError(errorCode)) goto _output_error;
+ CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) );
DISPLAYLEVEL(3, "Single Pass decompression : ");
- { size_t const decompressError = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL);
- if (LZ4F_isError(decompressError)) goto _output_error; }
+ CHECK( LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL) );
{ U64 const crcDest = XXH64(decodedBuffer, decodedBufferSize, 1);
if (crcDest != crcOrig) goto _output_error; }
DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedBufferSize);
@@ -257,8 +252,7 @@ int basicTests(U32 seed, double compressibility)
BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
size_t decResult, oSize = COMPRESSIBLE_NOISE_LENGTH;
DISPLAYLEVEL(3, "Missing last %u bytes : ", (U32)missingBytes);
- decResult = LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL);
- if (LZ4F_isError(decResult)) goto _output_error;
+ CHECK_V(decResult, LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL));
if (decResult != missingBytes) {
DISPLAY("%u bytes missing != %u bytes requested \n", (U32)missingBytes, (U32)decResult);
goto _output_error;
@@ -282,9 +276,9 @@ int basicTests(U32 seed, double compressibility)
const BYTE* ip = (BYTE*)compressedBuffer;
DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : ");
- errorCode = LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL);
- if (LZ4F_isError(errorCode)) goto _output_error;
- DISPLAYLEVEL(3, " %u \n", (unsigned)errorCode);
+ CHECK( LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL) );
+ //DISPLAYLEVEL(3, " %u \n", (unsigned)errorCode);
+ DISPLAYLEVEL(3, " OK \n");
DISPLAYLEVEL(3, "LZ4F_getFrameInfo on zero-size input : ");
{ size_t nullSize = 0;
@@ -309,8 +303,7 @@ int basicTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "LZ4F_getFrameInfo on enough input : ");
iSize = 15 - iSize;
- errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
- if (LZ4F_isError(errorCode)) goto _output_error;
+ CHECK( LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize) );
DISPLAYLEVEL(3, " correctly decoded \n");
ip += iSize;
}
@@ -342,8 +335,7 @@ int basicTests(U32 seed, double compressibility)
while (ip < iend) {
size_t oSize = oend-op;
size_t iSize = 1;
- errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
- if (LZ4F_isError(errorCode)) goto _output_error;
+ CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
op += oSize;
ip += iSize;
}
@@ -356,21 +348,18 @@ int basicTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "Using 64 KB block : ");
prefs.frameInfo.blockSizeID = LZ4F_max64KB;
prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
- cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
- if (LZ4F_isError(cSize)) goto _output_error;
+ CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs));
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "without checksum : ");
prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
- cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
- if (LZ4F_isError(cSize)) goto _output_error;
+ CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs));
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "Using 256 KB block : ");
prefs.frameInfo.blockSizeID = LZ4F_max256KB;
prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
- cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
- if (LZ4F_isError(cSize)) goto _output_error;
+ CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs));
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "Decompression test : \n");
@@ -388,8 +377,7 @@ int basicTests(U32 seed, double compressibility)
size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
size_t oSize = oend-op;
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
- { size_t const decompressError = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
- if (LZ4F_isError(decompressError)) goto _output_error; }
+ CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
op += oSize;
ip += iSize;
}
@@ -399,28 +387,24 @@ int basicTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
}
- { LZ4F_errorCode_t const freeError = LZ4F_freeDecompressionContext(dCtx);
- if (LZ4F_isError(freeError)) goto _output_error; }
+ CHECK( LZ4F_freeDecompressionContext(dCtx) );
dCtx = NULL;
}
DISPLAYLEVEL(3, "without checksum : ");
prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
- cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
- if (LZ4F_isError(cSize)) goto _output_error;
+ CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "Using 1 MB block : ");
prefs.frameInfo.blockSizeID = LZ4F_max1MB;
prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
- cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
- if (LZ4F_isError(cSize)) goto _output_error;
+ CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
- DISPLAYLEVEL(3, "without checksum : ");
+ DISPLAYLEVEL(3, "without frame checksum : ");
prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
- cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
- if (LZ4F_isError(cSize)) goto _output_error;
+ CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "Using 4 MB block : ");
@@ -428,70 +412,244 @@ int basicTests(U32 seed, double compressibility)
prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
{ size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs);
DISPLAYLEVEL(4, "dstCapacity = %u ; ", (U32)dstCapacity)
- cSize = LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs);
- if (LZ4F_isError(cSize)) goto _output_error;
+ CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs) );
DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
}
- DISPLAYLEVEL(3, "without checksum : ");
+ DISPLAYLEVEL(3, "without frame checksum : ");
prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
{ size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs);
DISPLAYLEVEL(4, "dstCapacity = %u ; ", (U32)dstCapacity)
- cSize = LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs);
- if (LZ4F_isError(cSize)) goto _output_error;
+ CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs) );
DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
}
- { size_t errorCode;
+ DISPLAYLEVEL(3, "LZ4F_compressFrame with block checksum : ");
+ memset(&prefs, 0, sizeof(prefs));
+ prefs.frameInfo.blockChecksumFlag = LZ4F_blockChecksumEnabled;
+ CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
+ DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
+
+ DISPLAYLEVEL(3, "Decompress with block checksum : ");
+ { size_t iSize = cSize;
+ size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
+ LZ4F_decompressionContext_t dctx;
+ CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
+ CHECK( LZ4F_decompress(dctx, decodedBuffer, &decodedSize, compressedBuffer, &iSize, NULL) );
+ if (decodedSize != testSize) goto _output_error;
+ if (iSize != cSize) goto _output_error;
+ { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 1);
+ U64 const crcSrc = XXH64(CNBuffer, testSize, 1);
+ if (crcDest != crcSrc) goto _output_error;
+ }
+ DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
+
+ CHECK( LZ4F_freeDecompressionContext(dctx) );
+ }
+
+ /* frame content size tests */
+ { size_t cErr;
BYTE* const ostart = (BYTE*)compressedBuffer;
BYTE* op = ostart;
- errorCode = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION);
- if (LZ4F_isError(errorCode)) goto _output_error;
+ CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
DISPLAYLEVEL(3, "compress without frameSize : ");
memset(&(prefs.frameInfo), 0, sizeof(prefs.frameInfo));
- errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs);
- if (LZ4F_isError(errorCode)) goto _output_error;
- op += errorCode;
- errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL);
- if (LZ4F_isError(errorCode)) goto _output_error;
- op += errorCode;
- errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL);
- if (LZ4F_isError(errorCode)) goto _output_error;
+ CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs));
+ op += cErr;
+ CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL));
+ op += cErr;
+ CHECK( LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL) );
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart));
DISPLAYLEVEL(3, "compress with frameSize : ");
prefs.frameInfo.contentSize = testSize;
op = ostart;
- errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs);
- if (LZ4F_isError(errorCode)) goto _output_error;
- op += errorCode;
- errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL);
- if (LZ4F_isError(errorCode)) goto _output_error;
- op += errorCode;
- errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL);
- if (LZ4F_isError(errorCode)) goto _output_error;
+ CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs));
+ op += cErr;
+ CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL));
+ op += cErr;
+ CHECK( LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL) );
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart));
DISPLAYLEVEL(3, "compress with wrong frameSize : ");
prefs.frameInfo.contentSize = testSize+1;
op = ostart;
- errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs);
- if (LZ4F_isError(errorCode)) goto _output_error;
- op += errorCode;
- errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL);
- if (LZ4F_isError(errorCode)) goto _output_error;
- op += errorCode;
- errorCode = LZ4F_compressEnd(cctx, op, testSize, NULL);
- if (LZ4F_isError(errorCode)) { DISPLAYLEVEL(3, "Error correctly detected : %s \n", LZ4F_getErrorName(errorCode)); }
- else
- goto _output_error;
-
- errorCode = LZ4F_freeCompressionContext(cctx);
- if (LZ4F_isError(errorCode)) goto _output_error;
+ CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs));
+ op += cErr;
+ CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL));
+ op += cErr;
+ cErr = LZ4F_compressEnd(cctx, op, testSize, NULL);
+ if (!LZ4F_isError(cErr)) goto _output_error;
+ DISPLAYLEVEL(3, "Error correctly detected : %s \n", LZ4F_getErrorName(cErr));
+
+ CHECK( LZ4F_freeCompressionContext(cctx) );
cctx = NULL;
}
+ /* dictID tests */
+ { size_t cErr;
+ U32 const dictID = 0x99;
+ CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
+
+ DISPLAYLEVEL(3, "insert a dictID : ");
+ memset(&prefs.frameInfo, 0, sizeof(prefs.frameInfo));
+ prefs.frameInfo.dictID = dictID;
+ CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs));
+ DISPLAYLEVEL(3, "created frame header of size %i bytes \n", (int)cErr);
+
+ DISPLAYLEVEL(3, "read a dictID : ");
+ CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) );
+ memset(&prefs.frameInfo, 0, sizeof(prefs.frameInfo));
+ CHECK( LZ4F_getFrameInfo(dCtx, &prefs.frameInfo, compressedBuffer, &cErr) );
+ if (prefs.frameInfo.dictID != dictID) goto _output_error;
+ DISPLAYLEVEL(3, "%u \n", (U32)prefs.frameInfo.dictID);
+
+ CHECK( LZ4F_freeDecompressionContext(dCtx) ); dCtx = NULL;
+ CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL;
+ }
+
+
+ /* Dictionary compression test */
+ { size_t const dictSize = 63 KB;
+ size_t const dstCapacity = LZ4F_compressFrameBound(dictSize, NULL);
+ size_t cSizeNoDict, cSizeWithDict;
+ LZ4F_CDict* const cdict = LZ4F_createCDict(CNBuffer, dictSize);
+ if (cdict == NULL) goto _output_error;
+ DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with NULL dict : ");
+ CHECK_V(cSizeNoDict,
+ LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity,
+ CNBuffer, dictSize,
+ NULL, NULL) );
+ DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeNoDict);
+
+ DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict : ");
+ CHECK_V(cSizeWithDict,
+ LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity,
+ CNBuffer, dictSize,
+ cdict, NULL) );
+ DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
+ (unsigned)dictSize, (unsigned)cSizeWithDict);
+ if (cSizeWithDict >= cSizeNoDict) goto _output_error; /* must be more efficient */
+ crcOrig = XXH64(CNBuffer, dictSize, 0);
+
+ DISPLAYLEVEL(3, "LZ4F_decompress_usingDict : ");
+ { LZ4F_dctx* dctx;
+ size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
+ size_t compressedSize = cSizeWithDict;
+ CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
+ CHECK( LZ4F_decompress_usingDict(dctx,
+ decodedBuffer, &decodedSize,
+ compressedBuffer, &compressedSize,
+ CNBuffer, dictSize,
+ NULL) );
+ if (compressedSize != cSizeWithDict) goto _output_error;
+ if (decodedSize != dictSize) goto _output_error;
+ { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0);
+ if (crcDest != crcOrig) goto _output_error; }
+ DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
+ CHECK( LZ4F_freeDecompressionContext(dctx) );
+ }
+
+ DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict, negative level : ");
+ { size_t cSizeLevelMax;
+ LZ4F_preferences_t cParams;
+ memset(&cParams, 0, sizeof(cParams));
+ cParams.compressionLevel = -3;
+ CHECK_V(cSizeLevelMax,
+ LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity,
+ CNBuffer, dictSize,
+ cdict, &cParams) );
+ DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax);
+ }
+
+ DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict, level max : ");
+ { size_t cSizeLevelMax;
+ LZ4F_preferences_t cParams;
+ memset(&cParams, 0, sizeof(cParams));
+ cParams.compressionLevel = LZ4F_compressionLevel_max();
+ CHECK_V(cSizeLevelMax,
+ LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity,
+ CNBuffer, dictSize,
+ cdict, &cParams) );
+ DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax);
+ }
+
+ DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, multiple linked blocks : ");
+ { size_t cSizeContiguous;
+ size_t const inSize = dictSize * 3;
+ size_t const outCapacity = LZ4F_compressFrameBound(inSize, NULL);
+ LZ4F_preferences_t cParams;
+ memset(&cParams, 0, sizeof(cParams));
+ cParams.frameInfo.blockMode = LZ4F_blockLinked;
+ cParams.frameInfo.blockSizeID = LZ4F_max64KB;
+ CHECK_V(cSizeContiguous,
+ LZ4F_compressFrame_usingCDict(compressedBuffer, outCapacity,
+ CNBuffer, inSize,
+ cdict, &cParams) );
+ DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
+ (unsigned)inSize, (unsigned)cSizeContiguous);
+
+ DISPLAYLEVEL(3, "LZ4F_decompress_usingDict on multiple linked blocks : ");
+ { LZ4F_dctx* dctx;
+ size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
+ size_t compressedSize = cSizeContiguous;
+ CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
+ CHECK( LZ4F_decompress_usingDict(dctx,
+ decodedBuffer, &decodedSize,
+ compressedBuffer, &compressedSize,
+ CNBuffer, dictSize,
+ NULL) );
+ if (compressedSize != cSizeContiguous) goto _output_error;
+ if (decodedSize != inSize) goto _output_error;
+ crcOrig = XXH64(CNBuffer, inSize, 0);
+ { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0);
+ if (crcDest != crcOrig) goto _output_error; }
+ DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
+ CHECK( LZ4F_freeDecompressionContext(dctx) );
+ }
+ }
+
+
+ DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, multiple independent blocks : ");
+ { size_t cSizeIndep;
+ size_t const inSize = dictSize * 3;
+ size_t const outCapacity = LZ4F_compressFrameBound(inSize, NULL);
+ LZ4F_preferences_t cParams;
+ memset(&cParams, 0, sizeof(cParams));
+ cParams.frameInfo.blockMode = LZ4F_blockIndependent;
+ cParams.frameInfo.blockSizeID = LZ4F_max64KB;
+ CHECK_V(cSizeIndep,
+ LZ4F_compressFrame_usingCDict(compressedBuffer, outCapacity,
+ CNBuffer, inSize,
+ cdict, &cParams) );
+ DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
+ (unsigned)inSize, (unsigned)cSizeIndep);
+
+ DISPLAYLEVEL(3, "LZ4F_decompress_usingDict on multiple independent blocks : ");
+ { LZ4F_dctx* dctx;
+ size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
+ size_t compressedSize = cSizeIndep;
+ CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
+ CHECK( LZ4F_decompress_usingDict(dctx,
+ decodedBuffer, &decodedSize,
+ compressedBuffer, &compressedSize,
+ CNBuffer, dictSize,
+ NULL) );
+ if (compressedSize != cSizeIndep) goto _output_error;
+ if (decodedSize != inSize) goto _output_error;
+ crcOrig = XXH64(CNBuffer, inSize, 0);
+ { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0);
+ if (crcDest != crcOrig) goto _output_error; }
+ DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
+ CHECK( LZ4F_freeDecompressionContext(dctx) );
+ }
+ }
+
+ LZ4F_freeCDict(cdict);
+ }
+
+
DISPLAYLEVEL(3, "Skippable frame test : \n");
{ size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
unsigned maxBits = FUZ_highbit((U32)decodedBufferSize);
@@ -500,8 +658,7 @@ int basicTests(U32 seed, double compressibility)
BYTE* ip = (BYTE*)compressedBuffer;
BYTE* iend = (BYTE*)compressedBuffer + cSize + 8;
- LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
- if (LZ4F_isError(errorCode)) goto _output_error;
+ CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) );
/* generate skippable frame */
FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START);
@@ -513,8 +670,7 @@ int basicTests(U32 seed, double compressibility)
size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
size_t oSize = oend-op;
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
- errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
- if (LZ4F_isError(errorCode)) goto _output_error;
+ CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
op += oSize;
ip += iSize;
}
@@ -533,8 +689,7 @@ int basicTests(U32 seed, double compressibility)
size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
size_t oSize = oend-op;
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
- errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
- if (LZ4F_isError(errorCode)) goto _output_error;
+ CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
op += oSize;
ip += iSize;
}
@@ -550,8 +705,7 @@ int basicTests(U32 seed, double compressibility)
size_t iSize = 10;
size_t oSize = 10;
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
- errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
- if (LZ4F_isError(errorCode)) goto _output_error;
+ CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
op += oSize;
ip += iSize;
}
@@ -603,6 +757,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
size_t result;
clock_t const startClock = clock();
clock_t const clockDuration = duration_s * CLOCKS_PER_SEC;
+# undef CHECK
# define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
@@ -639,6 +794,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
memset(&prefs, 0, sizeof(prefs));
prefs.frameInfo.blockMode = (LZ4F_blockMode_t)(FUZ_rand(&randState) & 1);
prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)(4 + (FUZ_rand(&randState) & 3));
+ prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)(FUZ_rand(&randState) & 1);
prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)(FUZ_rand(&randState) & 1);
prefs.frameInfo.contentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0;
prefs.autoFlush = neverFlush ? 0 : (FUZ_rand(&randState) & 7) == 2;