diff options
author | Yann Collet <yann.collet.73@gmail.com> | 2014-06-17 20:41:59 (GMT) |
---|---|---|
committer | Yann Collet <yann.collet.73@gmail.com> | 2014-06-17 20:41:59 (GMT) |
commit | d517d609d95bdbab665a6ddb6e018c450d1e5ae6 (patch) | |
tree | 9fe008ae09b4dcf79892952332f3738aa761760b /programs/lz4io.c | |
parent | 61ee86b5b831ad232f90302e24269abb9cc6db8f (diff) | |
download | lz4-d517d609d95bdbab665a6ddb6e018c450d1e5ae6.zip lz4-d517d609d95bdbab665a6ddb6e018c450d1e5ae6.tar.gz lz4-d517d609d95bdbab665a6ddb6e018c450d1e5ae6.tar.bz2 |
Fixed : streaming compression using small (<64KB) dictionary buffers
Diffstat (limited to 'programs/lz4io.c')
-rwxr-xr-x[-rw-r--r--] | programs/lz4io.c | 64 |
1 files changed, 24 insertions, 40 deletions
diff --git a/programs/lz4io.c b/programs/lz4io.c index 650681b..b581c41 100644..100755 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -121,7 +121,7 @@ #define CACHELINE 64 #define LEGACY_BLOCKSIZE (8 MB) -#define MIN_STREAM_BUFSIZE (1 MB + 64 KB) +#define MIN_STREAM_BUFSIZE (192 KB) #define LZ4S_BLOCKSIZEID_DEFAULT 7 #define LZ4S_CHECKSUM_SEED 0 #define LZ4S_EOS 0 @@ -412,7 +412,7 @@ static int compress_file_blockDependency(char* input_filename, char* output_file { initFunction = LZ4IO_LZ4_createStream; compressionFunction = LZ4IO_LZ4_compress_limitedOutput_continue; - nextBlockFunction = LZ4_moveDict; + nextBlockFunction = LZ4_saveDict; freeFunction = LZ4_free; } else @@ -502,7 +502,7 @@ static int compress_file_blockDependency(char* input_filename, char* output_file { size_t sizeToMove = 64 KB; if (inSize < 64 KB) sizeToMove = inSize; - nextBlockFunction(ctx, in_blockStart - sizeToMove, sizeToMove); + nextBlockFunction(ctx, in_blockStart - sizeToMove, (int)sizeToMove); if (compressionlevel>=3) in_blockStart = in_buff + 64 KB; } } @@ -749,8 +749,12 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) size_t sizeCheck; int blockChecksumFlag, streamChecksumFlag, blockIndependenceFlag; void* streamChecksumState=NULL; - int (*decompressionFunction)(const char*, char*, int, int) = LZ4_decompress_safe; - unsigned int prefix64k = 0; + int (*decompressionFunction)(void* ctx, const char* src, char* dst, int cSize, int maxOSize) = LZ4_decompress_safe_continue; + LZ4_streamDecode_t ctx; + + // init + memset(&ctx, 0, sizeof(ctx)); + (void)blockIndependenceFlag; // Decode stream descriptor nbReadBytes = fread(descriptor, 1, 3, finput); @@ -786,23 +790,17 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) if (checkBits != checkBits_xxh32) EXM_THROW(69, "Stream descriptor error detected"); } - if (!blockIndependenceFlag) - { - decompressionFunction = LZ4_decompress_safe_withPrefix64k; - prefix64k = 64 KB; - } - // Allocate Memory { - unsigned int outbuffSize = prefix64k+maxBlockSize; + size_t outBuffSize = maxBlockSize + 64 KB; + if (outBuffSize < MIN_STREAM_BUFSIZE) outBuffSize = MIN_STREAM_BUFSIZE; in_buff = (char*)malloc(maxBlockSize); - if (outbuffSize < MIN_STREAM_BUFSIZE) outbuffSize = MIN_STREAM_BUFSIZE; - out_buff = (char*)malloc(outbuffSize); - out_end = out_buff + outbuffSize; - out_start = out_buff + prefix64k; + out_buff = (char*)malloc(outBuffSize); + out_start = out_buff; + out_end = out_start + outBuffSize; if (!in_buff || !out_buff) EXM_THROW(70, "Allocation error : not enough memory"); + if (streamChecksumFlag) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED); } - if (streamChecksumFlag) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED); // Main Loop while (1) @@ -840,25 +838,19 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) if (sizeCheck != (size_t)blockSize) EXM_THROW(76, "Write error : cannot write data block"); filesize += blockSize; if (streamChecksumFlag) XXH32_update(streamChecksumState, in_buff, blockSize); - if (!blockIndependenceFlag) + if (!independentBlocks) { - if (blockSize >= prefix64k) - { - memcpy(out_buff, in_buff + (blockSize - prefix64k), prefix64k); // Required for reference for next blocks - out_start = out_buff + prefix64k; - continue; - } - else - { - memcpy(out_start, in_buff, blockSize); - decodedBytes = blockSize; - } + // handle dictionary for streaming + memcpy(in_buff + blockSize - 64 KB, out_buff, 64 KB); + LZ4_setDictDecode(&ctx, out_buff, 64 KB); + out_start = out_buff + 64 KB; } } else { // Decode Block - decodedBytes = decompressionFunction(in_buff, out_start, blockSize, maxBlockSize); + if (out_start + maxBlockSize > out_end) out_start = out_buff; + decodedBytes = decompressionFunction(&ctx, in_buff, out_start, blockSize, maxBlockSize); if (decodedBytes < 0) EXM_THROW(77, "Decoding Failed ! Corrupted input detected !"); filesize += decodedBytes; if (streamChecksumFlag) XXH32_update(streamChecksumState, out_start, decodedBytes); @@ -866,17 +858,9 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) // Write Block sizeCheck = fwrite(out_start, 1, decodedBytes, foutput); if (sizeCheck != (size_t)decodedBytes) EXM_THROW(78, "Write error : cannot write decoded block\n"); - } - - if (!blockIndependenceFlag) - { out_start += decodedBytes; - if ((size_t)(out_end - out_start) < (size_t)maxBlockSize) - { - memcpy(out_buff, out_start - prefix64k, prefix64k); - out_start = out_buff + prefix64k; - } } + } // Stream Checksum @@ -887,7 +871,7 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) sizeCheck = fread(&readChecksum, 1, 4, finput); if (sizeCheck != 4) EXM_THROW(74, "Read error : cannot read stream checksum"); readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian - if (checksum != readChecksum) EXM_THROW(75, "Error : invalid stream checksum detected"); + if (checksum != readChecksum) EXM_THROW(79, "Error : invalid stream checksum detected"); } // Free |