diff options
author | Yann Collet <Cyan4973@users.noreply.github.com> | 2018-02-01 16:53:23 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-02-01 16:53:23 (GMT) |
commit | 424c773f833796cbdbe784c05e6c9654fc28b741 (patch) | |
tree | b3d09a27df8edc9f12667f69a92d1f408d630e86 | |
parent | d1ccd620d62acb1863357aeb1bb64fe374b0d055 (diff) | |
parent | 3ce289bcce012b8173edd4c1040208da2692d060 (diff) | |
download | lz4-424c773f833796cbdbe784c05e6c9654fc28b741.zip lz4-424c773f833796cbdbe784c05e6c9654fc28b741.tar.gz lz4-424c773f833796cbdbe784c05e6c9654fc28b741.tar.bz2 |
Merge pull request #460 from lz4/frameCompress
refactored frameCompress.c example
-rw-r--r-- | Makefile | 5 | ||||
-rw-r--r-- | doc/lz4_manual.html | 4 | ||||
-rw-r--r-- | doc/lz4frame_manual.html | 44 | ||||
-rw-r--r-- | examples/Makefile | 44 | ||||
-rw-r--r-- | examples/frameCompress.c | 416 | ||||
-rw-r--r-- | lib/lz4frame.c | 3 | ||||
-rw-r--r-- | lib/lz4frame.h | 12 |
7 files changed, 308 insertions, 220 deletions
@@ -69,8 +69,8 @@ lz4 lz4-release : @cp $(PRGDIR)/lz4$(EXT) . .PHONY: examples -examples: lib lz4 - $(MAKE) -C $(EXDIR) test +examples: + $(MAKE) -C $(EXDIR) all .PHONY: manuals manuals: @@ -125,6 +125,7 @@ list: .PHONY: test test: $(MAKE) -C $(TESTDIR) $@ + $(MAKE) -C $(EXDIR) $@ clangtest: clean clang -v diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html index 6b7935d..46e6c3d 100644 --- a/doc/lz4_manual.html +++ b/doc/lz4_manual.html @@ -176,8 +176,8 @@ int LZ4_freeStream (LZ4_stream_t* streamPtr); If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. Important : Up to 64KB of previously compressed data is assumed to remain present and unmodified in memory ! - Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB. - Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. + Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB. + Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. @return : size of compressed block or 0 if there is an error (typically, compressed data cannot fit into 'dst') diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html index 590c632..1ddf6d8 100644 --- a/doc/lz4frame_manual.html +++ b/doc/lz4frame_manual.html @@ -101,8 +101,10 @@ <a name="Chapter5"></a><h2>Simple compression function</h2><pre></pre> <pre><b>size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr); -</b><p> Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences. - Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression. +</b><p> Returns the maximum possible compressed size with LZ4F_compressFrame() given srcSize and preferences. + `preferencesPtr` is optional. It can be replaced by NULL, in which case, the function will assume default preferences. + Note : this result is only usable with LZ4F_compressFrame(). + It may also be used with LZ4F_compressUpdate() _if no flush() operation_ is performed. </p></pre><BR> @@ -151,19 +153,21 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); </p></pre><BR> <pre><b>size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr); -</b><p> Provides dstCapacity given a srcSize to guarantee operation success in worst case situations. - prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario. - Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers. - When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. +</b><p> Provides minimum dstCapacity for a given srcSize to guarantee operation success in worst case scenarios. + Estimation includes frame footer, which would be generated by LZ4F_compressEnd(). + Estimation doesn't include frame header, already generated by LZ4F_compressBegin(). + prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario. + Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers. + When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. </p></pre><BR> <pre><b>size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr); -</b><p> LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. - An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations. - This value is provided by LZ4F_compressBound(). - If this condition is not respected, LZ4F_compress() will fail (result is an errorCode). - LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized. +</b><p> LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. + An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations. + This value is provided by LZ4F_compressBound(). + If this condition is not respected, LZ4F_compress() will fail (result is an errorCode). + LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized. `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). or an error code if it fails (which can be tested using LZ4F_isError()) @@ -171,8 +175,8 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); </p></pre><BR> <pre><b>size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr); -</b><p> When data must be generated and sent immediately, without waiting for a block to be completely filled, - it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx. +</b><p> When data must be generated and sent immediately, without waiting for a block to be completely filled, + it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx. `dstCapacity` must be large enough to ensure the operation will be successful. `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. @return : number of bytes written into dstBuffer (it can be zero, which means there was no data stored within cctx) @@ -200,13 +204,13 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); </b></pre><BR> <pre><b>LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version); LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx); -</b><p> Create an LZ4F_dctx object, to track all decompression operations. - The version provided MUST be LZ4F_VERSION. - The function provides a pointer to an allocated and initialized LZ4F_dctx object. - The result is an errorCode, which can be tested using LZ4F_isError(). - dctx memory can be released using LZ4F_freeDecompressionContext(); - The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released. - That is, it should be == 0 if decompression has been completed fully and correctly. +</b><p> Create an LZ4F_dctx object, to track all decompression operations. + The version provided MUST be LZ4F_VERSION. + The function provides a pointer to an allocated and initialized LZ4F_dctx object. + The result is an errorCode, which can be tested using LZ4F_isError(). + dctx memory can be released using LZ4F_freeDecompressionContext(); + The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released. + That is, it should be == 0 if decompression has been completed fully and correctly. </p></pre><BR> diff --git a/examples/Makefile b/examples/Makefile index 9321c24..f9e9e7a 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -52,34 +52,40 @@ default: all all: printVersion doubleBuffer dictionaryRandomAccess ringBuffer ringBufferHC \ lineCompress frameCompress simpleBuffer -printVersion: $(LZ4DIR)/lz4.c printVersion.c - $(CC) $(FLAGS) $^ -o $@$(EXT) +$(LZ4DIR)/liblz4.a: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4opt.h $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.h $(LZ4DIR)/lz4hc.h $(LZ4DIR)/lz4frame.h $(LZ4DIR)/lz4frame_static.h + $(MAKE) -C $(LZ4DIR) liblz4.a -doubleBuffer: $(LZ4DIR)/lz4.c blockStreaming_doubleBuffer.c - $(CC) $(FLAGS) $^ -o $@$(EXT) +printVersion: printVersion.c $(LZ4DIR)/liblz4.a + $(CC) $(FLAGS) $^ -o $@$(EXT) -dictionaryRandomAccess: $(LZ4DIR)/lz4.c dictionaryRandomAccess.c - $(CC) $(FLAGS) $^ -o $@$(EXT) +doubleBuffer: blockStreaming_doubleBuffer.c $(LZ4DIR)/liblz4.a + $(CC) $(FLAGS) $^ -o $@$(EXT) -ringBuffer : $(LZ4DIR)/lz4.c blockStreaming_ringBuffer.c - $(CC) $(FLAGS) $^ -o $@$(EXT) +dictionaryRandomAccess: dictionaryRandomAccess.c $(LZ4DIR)/liblz4.a + $(CC) $(FLAGS) $^ -o $@$(EXT) -ringBufferHC: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c HCStreaming_ringBuffer.c - $(CC) $(FLAGS) $^ -o $@$(EXT) +ringBuffer : blockStreaming_ringBuffer.c $(LZ4DIR)/liblz4.a + $(CC) $(FLAGS) $^ -o $@$(EXT) -lineCompress: $(LZ4DIR)/lz4.c blockStreaming_lineByLine.c - $(CC) $(FLAGS) $^ -o $@$(EXT) +ringBufferHC: HCStreaming_ringBuffer.c $(LZ4DIR)/liblz4.a + $(CC) $(FLAGS) $^ -o $@$(EXT) -frameCompress: frameCompress.c - $(CC) $(FLAGS) $^ -o $@$(EXT) $(LZ4DIR)/liblz4.a +lineCompress: blockStreaming_lineByLine.c $(LZ4DIR)/liblz4.a + $(CC) $(FLAGS) $^ -o $@$(EXT) -compressFunctions: $(LZ4DIR)/lz4.c compress_functions.c - $(CC) $(FLAGS) $^ -o $@$(EXT) -lrt +frameCompress: frameCompress.c $(LZ4DIR)/liblz4.a + $(CC) $(FLAGS) $^ -o $@$(EXT) -simpleBuffer: $(LZ4DIR)/lz4.c simple_buffer.c - $(CC) $(FLAGS) $^ -o $@$(EXT) +compressFunctions: compress_functions.c $(LZ4DIR)/liblz4.a + $(CC) $(FLAGS) $^ -o $@$(EXT) -lrt -test : all +simpleBuffer: simple_buffer.c $(LZ4DIR)/liblz4.a + $(CC) $(FLAGS) $^ -o $@$(EXT) + +$(LZ4) : + $(MAKE) -C ../programs lz4 + +test : all $(LZ4) @echo "\n=== Print Version ===" ./printVersion$(EXT) @echo "\n=== Simple compression example ===" diff --git a/examples/frameCompress.c b/examples/frameCompress.c index d66a8dc..972f716 100644 --- a/examples/frameCompress.c +++ b/examples/frameCompress.c @@ -1,122 +1,155 @@ -// LZ4frame API example : compress a file -// Based on sample code from Zbigniew Jędrzejewski-Szmek +/* LZ4frame API example : compress a file + * Modified from an example code by Zbigniew Jędrzejewski-Szmek + * + * This example streams an input file into an output file + * using a bounded memory budget. + * Input is read in chunks of IN_CHUNK_SIZE */ #include <stdio.h> #include <stdlib.h> #include <string.h> #include <errno.h> +#include <assert.h> #include <lz4frame.h> -#define BUF_SIZE 16*1024 -#define LZ4_HEADER_SIZE 19 -#define LZ4_FOOTER_SIZE 4 -static const LZ4F_preferences_t lz4_preferences = { +#define IN_CHUNK_SIZE (16*1024) + +static const LZ4F_preferences_t kPrefs = { { LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame, - 0 /* content size unknown */, 0 /* no dictID */ , LZ4F_noBlockChecksum }, - 0, /* compression level */ + 0 /* unknown content size */, 0 /* no dictID */ , LZ4F_noBlockChecksum }, + 0, /* compression level; 0 == default */ 0, /* autoflush */ { 0, 0, 0, 0 }, /* reserved, must be set to 0 */ }; -static size_t compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) { - size_t r=1; /* function result; 1 == error, default (early exit) */ - LZ4F_compressionContext_t ctx; - char *src, *buf = NULL; - size_t size, count_in = 0, count_out, offset = 0, frame_size; - if (LZ4F_isError( LZ4F_createCompressionContext(&ctx, LZ4F_VERSION) )) { - printf("Failed to create context: error %zu\n", r); - return 1; +/* safe_fwrite() : + * performs fwrite(), ensure operation success, or immediately exit() */ +static void safe_fwrite(void* buf, size_t eltSize, size_t nbElt, FILE* f) +{ + size_t const writtenSize = fwrite(buf, eltSize, nbElt, f); + size_t const expectedSize = eltSize * nbElt; /* note : should check for overflow */ + if (writtenSize < expectedSize) { + if (ferror(f)) /* note : ferror() must follow fwrite */ + printf("Write failed\n"); + else + printf("Short write\n"); + exit(1); } +} - src = malloc(BUF_SIZE); - if (!src) { - printf("Not enough memory\n"); - goto cleanup; - } - frame_size = LZ4F_compressBound(BUF_SIZE, &lz4_preferences); - size = frame_size + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE; - buf = malloc(size); - if (!buf) { - printf("Not enough memory\n"); - goto cleanup; - } +/* ================================================= */ +/* Streaming Compression example */ +/* ================================================= */ + +typedef struct { + int error; + unsigned long long size_in; + unsigned long long size_out; +} compressResult_t; + +static compressResult_t +compress_file_internal(FILE* f_in, FILE* f_out, + LZ4F_compressionContext_t ctx, + void* inBuff, size_t inChunkSize, + void* outBuff, size_t outCapacity) +{ + compressResult_t result = { 1, 0, 0 }; /* result for an error */ + unsigned long long count_in = 0, count_out; + + assert(f_in != NULL); assert(f_out != NULL); + assert(ctx != NULL); + assert(outCapacity >= LZ4F_HEADER_SIZE_MAX); + assert(outCapacity >= LZ4F_compressBound(inChunkSize, &kPrefs)); - { size_t const headerSize = LZ4F_compressBegin(ctx, buf, size, &lz4_preferences); + /* write frame header */ + { size_t const headerSize = LZ4F_compressBegin(ctx, outBuff, outCapacity, &kPrefs); if (LZ4F_isError(headerSize)) { printf("Failed to start compression: error %zu\n", headerSize); - goto cleanup; + return result; } - offset = count_out = headerSize; - printf("Buffer size is %zu bytes, header size %zu bytes\n", size, headerSize); + count_out = headerSize; + printf("Buffer size is %zu bytes, header size %zu bytes\n", outCapacity, headerSize); + safe_fwrite(outBuff, 1, headerSize, f_out); } - + /* stream file */ for (;;) { - size_t const readSize = fread(src, 1, BUF_SIZE, in); - if (readSize == 0) - break; + size_t const readSize = fread(inBuff, 1, IN_CHUNK_SIZE, f_in); + if (readSize == 0) break; /* nothing left to read from input file */ count_in += readSize; - { size_t const compressedSize = LZ4F_compressUpdate(ctx, buf + offset, size - offset, src, readSize, NULL); - if (LZ4F_isError(compressedSize)) { - printf("Compression failed: error %zu\n", compressedSize); - goto cleanup; - } - offset += compressedSize; - count_out += compressedSize; + size_t const compressedSize = LZ4F_compressUpdate(ctx, + outBuff, outCapacity, + inBuff, readSize, + NULL); + if (LZ4F_isError(compressedSize)) { + printf("Compression failed: error %zu\n", compressedSize); + return result; } - if (size - offset < frame_size + LZ4_FOOTER_SIZE) { - size_t writtenSize; - printf("Writing %zu bytes\n", offset); - - writtenSize = fwrite(buf, 1, offset, out); - if (writtenSize < offset) { - if (ferror(out)) /* note : ferror() must follow fwrite */ - printf("Write failed\n"); - else - printf("Short write\n"); - goto cleanup; - } - - offset = 0; - } + printf("Writing %zu bytes\n", compressedSize); + safe_fwrite(outBuff, 1, compressedSize, f_out); + count_out += compressedSize; } - { size_t const compressedSize = LZ4F_compressEnd(ctx, buf + offset, size - offset, NULL); + /* flush whatever remains within internal buffers */ + { size_t const compressedSize = LZ4F_compressEnd(ctx, + outBuff, outCapacity, + NULL); if (LZ4F_isError(compressedSize)) { printf("Failed to end compression: error %zu\n", compressedSize); - goto cleanup; + return result; } - offset += compressedSize; + + printf("Writing %zu bytes\n", compressedSize); + safe_fwrite(outBuff, 1, compressedSize, f_out); count_out += compressedSize; } - printf("Writing %zu bytes\n", offset); - { size_t const writtenSize = fwrite(buf, 1, offset, out); - if (writtenSize < offset) { - if (ferror(out)) - printf("Write failed\n"); - else - printf("Short write\n"); - goto cleanup; - } } + result.size_in = count_in; + result.size_out = count_out; + result.error = 0; + return result; +} + +static compressResult_t +compress_file(FILE* f_in, FILE* f_out) +{ + assert(f_in != NULL); + assert(f_out != NULL); - *size_in = count_in; - *size_out = count_out; - r = 0; /* success */ + /* ressource allocation */ + LZ4F_compressionContext_t ctx; + size_t const ctxCreation = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); + void* const src = malloc(IN_CHUNK_SIZE); + size_t const outbufCapacity = LZ4F_compressBound(IN_CHUNK_SIZE, &kPrefs); /* large enough for any input <= IN_CHUNK_SIZE */ + void* const outbuff = malloc(outbufCapacity); + + compressResult_t result = { 1, 0, 0 }; /* == error (default) */ + if (!LZ4F_isError(ctxCreation) && src && outbuff) { + result = compress_file_internal(f_in, f_out, + ctx, + src, IN_CHUNK_SIZE, + outbuff, outbufCapacity); + } else { + printf("error : ressource allocation failed \n"); + } - cleanup: LZ4F_freeCompressionContext(ctx); /* supports free on NULL */ free(src); - free(buf); - return r; + free(outbuff); + return result; } + +/* ================================================= */ +/* Streaming decompression example */ +/* ================================================= */ + static size_t get_block_size(const LZ4F_frameInfo_t* info) { switch (info->blockSizeID) { case LZ4F_default: @@ -125,125 +158,164 @@ static size_t get_block_size(const LZ4F_frameInfo_t* info) { case LZ4F_max1MB: return 1 << 20; case LZ4F_max4MB: return 1 << 22; default: - printf("Impossible unless more block sizes are allowed\n"); + printf("Impossible with expected frame specification (<=v1.6.1)\n"); exit(1); } } -static size_t decompress_file(FILE* in, FILE* out) { - void* const src = malloc(BUF_SIZE); - void* dst = NULL; - size_t dstCapacity = 0; - LZ4F_dctx* dctx = NULL; +/* @return : 1==error, 0==success */ +static int +decompress_file_internal(FILE* f_in, FILE* f_out, + LZ4F_dctx* dctx, + void* src, size_t srcCapacity, size_t filled, size_t alreadyConsumed, + void* dst, size_t dstCapacity) +{ + int firstChunk = 1; size_t ret = 1; - /* Initialization */ - if (!src) { perror("decompress_file(src)"); goto cleanup; } - { size_t const dctxStatus = LZ4F_createDecompressionContext(&dctx, 100); - if (LZ4F_isError(dctxStatus)) { - printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(dctxStatus)); - goto cleanup; - } } + assert(f_in != NULL); assert(f_out != NULL); + assert(dctx != NULL); + assert(src != NULL); assert(srcCapacity > 0); assert(filled <= srcCapacity); assert(alreadyConsumed <= filled); + assert(dst != NULL); assert(dstCapacity > 0); /* Decompression */ while (ret != 0) { /* Load more input */ - size_t srcSize = fread(src, 1, BUF_SIZE, in); - const void* srcPtr = src; - const void* const srcEnd = srcPtr + srcSize; - if (srcSize == 0 || ferror(in)) { + size_t readSize = firstChunk ? filled : fread(src, 1, srcCapacity, f_in); firstChunk=0; + const void* srcPtr = src + alreadyConsumed; alreadyConsumed=0; + const void* const srcEnd = srcPtr + readSize; + if (readSize == 0 || ferror(f_in)) { printf("Decompress: not enough input or error reading file\n"); - goto cleanup; - } - /* Allocate destination buffer if it isn't already */ - if (!dst) { - LZ4F_frameInfo_t info; - ret = LZ4F_getFrameInfo(dctx, &info, src, &srcSize); - if (LZ4F_isError(ret)) { - printf("LZ4F_getFrameInfo error: %s\n", LZ4F_getErrorName(ret)); - goto cleanup; - } - /* Allocating enough space for an entire block isn't necessary for - * correctness, but it allows some memcpy's to be elided. - */ - dstCapacity = get_block_size(&info); - dst = malloc(dstCapacity); - if (!dst) { perror("decompress_file(dst)"); goto cleanup; } - srcPtr += srcSize; - srcSize = srcEnd - srcPtr; + return 1; } + /* Decompress: - * Continue while there is more input to read and the frame isn't over. - * If srcPtr == srcEnd then we know that there is no more output left in the - * internal buffer left to flush. + * Continue while there is more input to read (srcPtr != srcEnd) + * and the frame isn't over (ret != 0) */ while (srcPtr != srcEnd && ret != 0) { - /* INVARIANT: Any data left in dst has already been written */ + /* Any data within dst has been flushed at this stage */ size_t dstSize = dstCapacity; + size_t srcSize = srcEnd - srcPtr; ret = LZ4F_decompress(dctx, dst, &dstSize, srcPtr, &srcSize, /* LZ4F_decompressOptions_t */ NULL); if (LZ4F_isError(ret)) { printf("Decompression error: %s\n", LZ4F_getErrorName(ret)); - goto cleanup; + return 1; } /* Flush output */ - if (dstSize != 0){ - size_t written = fwrite(dst, 1, dstSize, out); - printf("Writing %zu bytes\n", dstSize); - if (written != dstSize) { - printf("Decompress: Failed to write to file\n"); - goto cleanup; - } - } + if (dstSize != 0) safe_fwrite(dst, 1, dstSize, f_out); /* Update input */ srcPtr += srcSize; - srcSize = srcEnd - srcPtr; } } + /* Check that there isn't trailing input data after the frame. - * It is valid to have multiple frames in the same file, but this example - * doesn't support it. + * It is valid to have multiple frames in the same file, + * but this example only supports one frame. */ - ret = fread(src, 1, 1, in); - if (ret != 0 || !feof(in)) { - printf("Decompress: Trailing data left in file after frame\n"); - goto cleanup; + { size_t const readSize = fread(src, 1, 1, f_in); + if (readSize != 0 || !feof(f_in)) { + printf("Decompress: Trailing data left in file after frame\n"); + return 1; + } } + + return 0; +} + + +/* @return : 1==error, 0==completed */ +static int +decompress_file_allocDst(FILE* f_in, FILE* f_out, + LZ4F_dctx* dctx, + void* src, size_t srcCapacity) +{ + assert(f_in != NULL); assert(f_out != NULL); + assert(dctx != NULL); + assert(src != NULL); + assert(srcCapacity >= LZ4F_HEADER_SIZE_MAX); /* ensure LZ4F_getFrameInfo() can read enough data */ + + /* Read Frame header */ + size_t const readSize = fread(src, 1, srcCapacity, f_in); + if (readSize == 0 || ferror(f_in)) { + printf("Decompress: not enough input or error reading file\n"); + return 1; } -cleanup: - free(src); + LZ4F_frameInfo_t info; + size_t consumedSize = readSize; + { size_t const fires = LZ4F_getFrameInfo(dctx, &info, src, &consumedSize); + if (LZ4F_isError(fires)) { + printf("LZ4F_getFrameInfo error: %s\n", LZ4F_getErrorName(fires)); + return 1; + } } + + /* Allocating enough space for an entire block isn't necessary for + * correctness, but it allows some memcpy's to be elided. + */ + size_t const dstCapacity = get_block_size(&info); + void* const dst = malloc(dstCapacity); + if (!dst) { perror("decompress_file(dst)"); return 1; } + + int const decompressionResult = decompress_file_internal( + f_in, f_out, + dctx, + src, srcCapacity, readSize, consumedSize, + dst, dstCapacity); + free(dst); - return LZ4F_freeDecompressionContext(dctx); /* note : free works on NULL */ + return decompressionResult; +} + + +/* @result : 1==error, 0==success */ +static int decompress_file(FILE* f_in, FILE* f_out) +{ + assert(f_in != NULL); assert(f_out != NULL); + + /* Ressource allocation */ + void* const src = malloc(IN_CHUNK_SIZE); + if (!src) { perror("decompress_file(src)"); return 1; } + + LZ4F_dctx* dctx; + { size_t const dctxStatus = LZ4F_createDecompressionContext(&dctx, 100); + if (LZ4F_isError(dctxStatus)) { + printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(dctxStatus)); + } } + + int const result = !dctx ? 1 /* error */ : + decompress_file_allocDst(f_in, f_out, dctx, src, IN_CHUNK_SIZE); + + free(src); + LZ4F_freeDecompressionContext(dctx); /* note : free works on NULL */ + return result; } -int compare(FILE* fp0, FILE* fp1) + +int compareFiles(FILE* fp0, FILE* fp1) { int result = 0; - while(0 == result) { + while (result==0) { char b0[1024]; char b1[1024]; - const size_t r0 = fread(b0, 1, sizeof(b0), fp0); - const size_t r1 = fread(b1, 1, sizeof(b1), fp1); - - result = (int) r0 - (int) r1; + size_t const r0 = fread(b0, 1, sizeof(b0), fp0); + size_t const r1 = fread(b1, 1, sizeof(b1), fp1); - if (0 == r0 || 0 == r1) { - break; - } - if (0 == result) { - result = memcmp(b0, b1, r0); - } + result = (r0 != r1); + if (!r0 || !r1) break; + if (!result) result = memcmp(b0, b1, r0); } return result; } + int main(int argc, const char **argv) { char inpFilename[256] = { 0 }; char lz4Filename[256] = { 0 }; char decFilename[256] = { 0 }; - if(argc < 2) { + if (argc < 2) { printf("Please specify input filename\n"); return 0; } @@ -259,40 +331,39 @@ int main(int argc, const char **argv) { /* compress */ { FILE* const inpFp = fopen(inpFilename, "rb"); FILE* const outFp = fopen(lz4Filename, "wb"); - size_t sizeIn = 0; - size_t sizeOut = 0; - size_t ret; printf("compress : %s -> %s\n", inpFilename, lz4Filename); - ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut); - if (ret) { - printf("compress : failed with code %zu\n", ret); - return (int)ret; - } - printf("%s: %zu → %zu bytes, %.1f%%\n", - inpFilename, sizeIn, sizeOut, - (double)sizeOut / sizeIn * 100); - printf("compress : done\n"); + compressResult_t const ret = compress_file(inpFp, outFp); fclose(outFp); fclose(inpFp); + + if (ret.error) { + printf("compress : failed with code %i\n", ret.error); + return ret.error; + } + printf("%s: %zu → %zu bytes, %.1f%%\n", + inpFilename, + (size_t)ret.size_in, (size_t)ret.size_out, /* might overflow is size_t is 32 bits and size_{in,out} > 4 GB */ + (double)ret.size_out / ret.size_in * 100); + printf("compress : done\n"); } /* decompress */ { FILE* const inpFp = fopen(lz4Filename, "rb"); FILE* const outFp = fopen(decFilename, "wb"); - size_t ret; printf("decompress : %s -> %s\n", lz4Filename, decFilename); - ret = decompress_file(inpFp, outFp); - if (ret) { - printf("decompress : failed with code %zu\n", ret); - return (int)ret; - } - printf("decompress : done\n"); + int const ret = decompress_file(inpFp, outFp); fclose(outFp); fclose(inpFp); + + if (ret) { + printf("decompress : failed with code %i\n", ret); + return ret; + } + printf("decompress : done\n"); } /* verify */ @@ -300,15 +371,16 @@ int main(int argc, const char **argv) { FILE* const decFp = fopen(decFilename, "rb"); printf("verify : %s <-> %s\n", inpFilename, decFilename); - const int cmp = compare(inpFp, decFp); - if(0 == cmp) { - printf("verify : OK\n"); - } else { - printf("verify : NG\n"); - } + int const cmp = compareFiles(inpFp, decFp); fclose(decFp); fclose(inpFp); + + if (cmp) { + printf("corruption detected : decompressed file differs from original\n"); + return cmp; + } + printf("verify : OK\n"); } return 0; diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 0b26f75..a394d1f 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -726,7 +726,8 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC); - if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize)) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); + if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize)) + return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); memset(&cOptionsNull, 0, sizeof(cOptionsNull)); if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; diff --git a/lib/lz4frame.h b/lib/lz4frame.h index eb55e45..15484d7 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -189,8 +189,10 @@ LZ4FLIB_API int LZ4F_compressionLevel_max(void); * Simple compression function ***********************************/ /*! LZ4F_compressFrameBound() : - * Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences. - * Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression. + * Returns the maximum possible compressed size with LZ4F_compressFrame() given srcSize and preferences. + * `preferencesPtr` is optional. It can be replaced by NULL, in which case, the function will assume default preferences. + * Note : this result is only usable with LZ4F_compressFrame(). + * It may also be used with LZ4F_compressUpdate() _if no flush() operation_ is performed. */ LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr); @@ -235,7 +237,7 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); /*---- Compression ----*/ -#define LZ4F_HEADER_SIZE_MAX 19 +#define LZ4F_HEADER_SIZE_MAX 19 /* LZ4 Frame header size can vary from 7 to 19 bytes */ /*! LZ4F_compressBegin() : * will write the frame header into dstBuffer. * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. @@ -248,7 +250,9 @@ LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, const LZ4F_preferences_t* prefsPtr); /*! LZ4F_compressBound() : - * Provides minimum dstCapacity for a given srcSize to guarantee operation success in worst case situations. + * Provides minimum dstCapacity for a given srcSize to guarantee operation success in worst case scenarios. + * Estimation includes frame footer, which would be generated by LZ4F_compressEnd(). + * Estimation doesn't include frame header, already generated by LZ4F_compressBegin(). * prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario. * Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers. * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. |