diff options
author | Yann Collet <yann.collet.73@gmail.com> | 2014-09-09 22:54:22 (GMT) |
---|---|---|
committer | Yann Collet <yann.collet.73@gmail.com> | 2014-09-09 22:54:22 (GMT) |
commit | eac83cd850d2f69a82ef9af344be9b7f3925681a (patch) | |
tree | 53597b544737bdb8345efa31487d31f1de544b4b /lz4frame.c | |
parent | 6b407c65a443e88c6cdd63dd738fbf94b9d6a886 (diff) | |
download | lz4-eac83cd850d2f69a82ef9af344be9b7f3925681a.zip lz4-eac83cd850d2f69a82ef9af344be9b7f3925681a.tar.gz lz4-eac83cd850d2f69a82ef9af344be9b7f3925681a.tar.bz2 |
Added : streaming mode
Diffstat (limited to 'lz4frame.c')
-rw-r--r-- | lz4frame.c | 156 |
1 files changed, 120 insertions, 36 deletions
@@ -118,9 +118,13 @@ typedef struct { unsigned cStage; size_t maxBlockSize; size_t maxBufferSize; - XXH32_stateSpace_t xxh; - BYTE* tmpIn; + BYTE* tmpBuff; + BYTE* tmpDict; + size_t tmpDictSize; + BYTE* tmpIn; size_t tmpInSize; + XXH32_stateSpace_t xxh; + LZ4_stream_t lz4ctx; } LZ4F_cctx_internal_t; typedef struct { @@ -128,16 +132,21 @@ typedef struct { unsigned version; unsigned dStage; size_t maxBlockSize; - XXH32_stateSpace_t xxh; + size_t maxBufferSize; size_t sizeToDecode; const BYTE* srcExpect; BYTE* tmpIn; size_t tmpInSize; size_t tmpInTarget; + BYTE* tmpOutBuffer; + BYTE* dict; + size_t dictSize; BYTE* tmpOut; size_t tmpOutSize; size_t tmpOutStart; - BYTE header[7]; + XXH32_stateSpace_t xxh; + LZ4_streamDecode_t lz4ctx; + BYTE header[8]; } LZ4F_dctx_internal_t; @@ -322,7 +331,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp { LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)LZ4F_compressionContext; - FREEMEM(cctxPtr->tmpIn); + FREEMEM(cctxPtr->tmpBuff); FREEMEM(LZ4F_compressionContext); return OK_NoError; @@ -351,15 +360,21 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds cctxPtr->prefs = *preferencesPtr; if (cctxPtr->prefs.frameInfo.blockSizeID == 0) cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; cctxPtr->maxBlockSize = LZ4F_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID); - if (cctxPtr->maxBufferSize < cctxPtr->maxBlockSize) + if (cctxPtr->maxBufferSize < cctxPtr->maxBlockSize + (cctxPtr->prefs.frameInfo.blockMode == blockLinked)) { cctxPtr->maxBufferSize = cctxPtr->maxBlockSize; - FREEMEM(cctxPtr->tmpIn); - cctxPtr->tmpIn = ALLOCATOR(cctxPtr->maxBlockSize); - if (cctxPtr->tmpIn == NULL) return -ERROR_allocation_failed; + if (cctxPtr->prefs.frameInfo.blockMode == blockLinked) cctxPtr->maxBufferSize += 64 KB; + FREEMEM(cctxPtr->tmpBuff); + cctxPtr->tmpBuff = ALLOCATOR(cctxPtr->maxBufferSize); + if (cctxPtr->tmpBuff == NULL) return -ERROR_allocation_failed; + cctxPtr->tmpDict = cctxPtr->tmpBuff; + cctxPtr->tmpIn = cctxPtr->tmpBuff; + if (cctxPtr->prefs.frameInfo.blockMode == blockLinked) cctxPtr->tmpIn += 64 KB; } + cctxPtr->tmpDictSize = 0; cctxPtr->tmpInSize = 0; XXH32_resetState(&(cctxPtr->xxh), 0); + LZ4_resetStream(&(cctxPtr->lz4ctx)); /* Magic Number */ LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER); @@ -367,18 +382,16 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds headerStart = dstPtr; /* FLG Byte */ - *dstPtr = (1 & _2BITS) << 6; /* Version('01') */ - *dstPtr |= (1 & _1BIT ) << 5; /* Blocks independents */ - *dstPtr |= (char)((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2); /* Stream checksum */ - dstPtr++; + //cctxPtr->prefs.frameInfo.blockMode = 1; // <============ debug + *dstPtr++ = ((1 & _2BITS) << 6) /* Version('01') */ + + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) /* Block mode */ + + (char)((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2); /* Stream checksum */ /* BD Byte */ - *dstPtr = (char)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4); - dstPtr++; + *dstPtr++ = (char)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4); /* CRC Byte */ - *dstPtr = LZ4F_headerChecksum(headerStart, 2); - dstPtr++; + *dstPtr++ = LZ4F_headerChecksum(headerStart, 2); - cctxPtr->cStage = 1; /* header written */ + cctxPtr->cStage = 1; /* header written, wait for data block */ return (dstPtr - dstStart); } @@ -442,34 +455,44 @@ size_t LZ4F_compress(LZ4F_compressionContext_t compressionContext, void* dstBuff const BYTE* const srcEnd = srcPtr + srcSize; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; + U32 lastBlockCompressed = 0; + int (*compress)(void*, const char*, char*, int, int); + if (cctxPtr->cStage != 1) return -ERROR_GENERIC; if (dstMaxSize < LZ4F_compressBound(srcSize, &(cctxPtr->prefs.frameInfo))) return -ERROR_dstMaxSize_tooSmall; if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; + /* select compression function */ + compress = (cctxPtr->prefs.frameInfo.blockMode == blockLinked) ? + (int(*)(void*,const char*,char*,int,int))LZ4_compress_limitedOutput_continue : + LZ4_compress_limitedOutput_withState; + /* complete tmp buffer */ - if (cctxPtr->tmpInSize > 0) + if (cctxPtr->tmpInSize > 0) /* some data already within tmp buffer */ { size_t sizeToCopy = blockSize - cctxPtr->tmpInSize; if (sizeToCopy > srcSize) { - /* add to tmp buffer */ + /* add src to tmpIn buffer */ memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize); srcPtr = srcEnd; cctxPtr->tmpInSize += srcSize; } else { + /* complete tmpIn block and then compress it */ BYTE* cSizePtr = dstPtr; U32 cSize; + lastBlockCompressed = 2; memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy); srcPtr += sizeToCopy; - dstPtr += 4; /* space for cSizePtr */ - cSize = (U32)LZ4_compress_limitedOutput((const char*)cctxPtr->tmpIn, (char*)dstPtr, (int)(blockSize), (int)(blockSize-1)); + dstPtr += 4; /* space for cSize */ + cSize = (U32)compress(&(cctxPtr->lz4ctx), (const char*)cctxPtr->tmpIn, (char*)dstPtr, (int)(blockSize), (int)(blockSize-1)); dstPtr += cSize; LZ4F_writeLE32(cSizePtr, cSize); - if (cSize == 0) /* compression failed */ + if (cSize == 0) /* compression failed : non compressible assumed */ { cSize = blockSize + LZ4F_BLOCKUNCOMPRESSED_FLAG; LZ4F_writeLE32(cSizePtr, cSize); @@ -485,8 +508,9 @@ size_t LZ4F_compress(LZ4F_compressionContext_t compressionContext, void* dstBuff /* compress one block */ BYTE* cSizePtr = dstPtr; U32 cSize; + lastBlockCompressed = 1; dstPtr += 4; /* space for cSizePtr */ - cSize = (U32)LZ4_compress_limitedOutput((const char*)srcPtr, (char*)dstPtr, (int)(blockSize), (int)(blockSize-1)); + cSize = (U32)compress(&(cctxPtr->lz4ctx), (const char*)srcPtr, (char*)dstPtr, (int)(blockSize), (int)(blockSize-1)); dstPtr += cSize; LZ4F_writeLE32(cSizePtr, cSize); if (cSize == 0) /* compression failed */ @@ -499,7 +523,16 @@ size_t LZ4F_compress(LZ4F_compressionContext_t compressionContext, void* dstBuff srcPtr += blockSize; } - if (srcPtr < srcEnd) + if ((cctxPtr->prefs.frameInfo.blockMode == blockLinked) && (lastBlockCompressed)) + { + /* last 64 KB of input become dictionary */ + /* assumption : previous block size was at least 64 KB */ + int result = LZ4_saveDict (&(cctxPtr->lz4ctx), (char*)(cctxPtr->tmpDict), 64 KB); + if (!result) return ERROR_GENERIC; + cctxPtr->tmpIn = cctxPtr->tmpDict + result; + } + + if (srcPtr < srcEnd) /* some input data left */ { /* fill tmp buffer */ size_t sizeToCopy = srcEnd - srcPtr; @@ -528,6 +561,7 @@ size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; + int (*compress)(void*, const char*, char*, int, int); if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */ @@ -535,11 +569,16 @@ size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, if (dstMaxSize < LZ4F_compressBound(1, &(cctxPtr->prefs.frameInfo))) return -ERROR_dstMaxSize_tooSmall; if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; + /* select compression function */ + compress = (cctxPtr->prefs.frameInfo.blockMode == blockLinked) ? + (int(*)(void*,const char*,char*,int,int))LZ4_compress_limitedOutput_continue : + LZ4_compress_limitedOutput_withState; + { BYTE* cSizePtr = dstPtr; U32 cSize; dstPtr += 4; /* space for cSizePtr */ - cSize = (U32)LZ4_compress_limitedOutput((const char*)cctxPtr->tmpIn, (char*)dstPtr, (int)(cctxPtr->tmpInSize), (int)(cctxPtr->tmpInSize-1)); + cSize = (U32)compress(&(cctxPtr->lz4ctx), (const char*)cctxPtr->tmpIn, (char*)dstPtr, (int)(cctxPtr->tmpInSize), (int)(cctxPtr->tmpInSize-1)); dstPtr += cSize; LZ4F_writeLE32(cSizePtr, cSize); if (cSize == 0) /* compression failed */ @@ -552,6 +591,14 @@ size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, cctxPtr->tmpInSize = 0; } + if (cctxPtr->prefs.frameInfo.blockMode == blockLinked) + { + /* last 64 KB of input become dictionary */ + int result = LZ4_saveDict (&(cctxPtr->lz4ctx), (char*)(cctxPtr->tmpDict), 64 KB); + if (!result) return ERROR_GENERIC; + cctxPtr->tmpIn = cctxPtr->tmpDict + result; + } + return dstPtr - dstStart; } @@ -620,7 +667,7 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_compressionContext_t LZ4F_de { LZ4F_dctx_internal_t* dctxPtr = (LZ4F_dctx_internal_t*)LZ4F_decompressionContext; FREEMEM(dctxPtr->tmpIn); - FREEMEM(dctxPtr->tmpOut); + FREEMEM(dctxPtr->tmpOutBuffer); FREEMEM(dctxPtr); return OK_NoError; } @@ -657,7 +704,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPt /* validate */ if (version != 1) return -ERROR_GENERIC; /* Version Number, only supported value */ - if (blockMode != blockIndependent) return -ERROR_GENERIC; /* Only supported blockMode for the time being */ + //if (blockMode != blockIndependent) return -ERROR_GENERIC; /* Only supported blockMode for the time being */ if (blockChecksumFlag != 0) return -ERROR_GENERIC; /* Only supported value for the time being */ if (contentSizeFlag != 0) return -ERROR_GENERIC; /* Only supported value for the time being */ if (((FLG>>1)&_1BIT) != 0) return -ERROR_GENERIC; /* Reserved bit */ @@ -673,16 +720,23 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPt /* init */ if (contentChecksumFlag) XXH32_resetState(&(dctxPtr->xxh), 0); + if (blockMode==blockLinked) LZ4_setStreamDecode(&(dctxPtr->lz4ctx), NULL, 0); + dctxPtr->dictSize = 0; - if (LZ4F_getBlockSize(blockSizeID) > dctxPtr->maxBlockSize) /* tmp buffers too small */ + if (LZ4F_getBlockSize(blockSizeID) > dctxPtr->maxBufferSize) /* tmp buffers too small */ { FREEMEM(dctxPtr->tmpIn); - FREEMEM(dctxPtr->tmpOut); - dctxPtr->tmpIn = ALLOCATOR(LZ4F_getBlockSize(blockSizeID)); - if (dctxPtr->tmpIn == NULL) return -ERROR_GENERIC; - dctxPtr->tmpOut= ALLOCATOR(LZ4F_getBlockSize(blockSizeID)); - if (dctxPtr->tmpOut== NULL) return -ERROR_GENERIC; + FREEMEM(dctxPtr->tmpOutBuffer); dctxPtr->maxBlockSize = LZ4F_getBlockSize(blockSizeID); + dctxPtr->maxBufferSize = dctxPtr->maxBlockSize; + if (dctxPtr->frameInfo.blockMode==blockLinked) dctxPtr->maxBufferSize += 64 KB; + dctxPtr->tmpIn = ALLOCATOR(dctxPtr->maxBlockSize); + if (dctxPtr->tmpIn == NULL) return -ERROR_GENERIC; + dctxPtr->tmpOutBuffer= ALLOCATOR(dctxPtr->maxBufferSize); + if (dctxPtr->tmpOutBuffer== NULL) return -ERROR_GENERIC; + dctxPtr->tmpOut = dctxPtr->tmpOutBuffer; + if (dctxPtr->frameInfo.blockMode==blockLinked) dctxPtr->tmpOut += 64 KB; + dctxPtr->dict = dctxPtr->tmpOut - dctxPtr->dictSize; } return 7; @@ -724,6 +778,27 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t decompressionCont } +static void LZ4F_saveDict(LZ4F_dctx_internal_t* dctxPtr, BYTE* decoded, size_t decodedSize) +{ + size_t newDictSize = decodedSize; + size_t preserveDictSize; + if (newDictSize > 64 KB) newDictSize = 64 KB; + preserveDictSize = 64 KB - newDictSize; + memmove(dctxPtr->tmpOutBuffer, dctxPtr->tmpOutBuffer + newDictSize, preserveDictSize); + memcpy(dctxPtr->tmpOutBuffer + preserveDictSize, decoded + decodedSize - newDictSize, newDictSize); + dctxPtr->dictSize += newDictSize; + if (dctxPtr->dictSize > 64 KB) dctxPtr->dictSize = 64 KB; + dctxPtr->dict = dctxPtr->tmpOut - dctxPtr->dictSize; +} + + +static int LZ4F_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize) +{ + (void)dictStart; (void)dictSize; + return LZ4_decompress_safe (source, dest, compressedSize, maxDecompressedSize); +} + + /* LZ4F_decompress() * Call this function repetitively to regenerate data compressed within srcBuffer. * The function will attempt to decode *srcSize from srcBuffer, into dstBuffer of maximum size *dstSize. @@ -750,6 +825,7 @@ LZ4F_errorCode_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContex size_t nextCBlockSize=0; const BYTE* selectedIn=NULL; LZ4F_errorCode_t goodResult = OK_NoError; + int (*decoder)(const char*, char*, int, int, const char*, int); if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull; @@ -890,21 +966,29 @@ goto_getCBlock: goto_decodeCBlock: { int decodedSize; + if (dctxPtr->frameInfo.blockMode == blockLinked) + decoder = LZ4_decompress_safe_usingDict; + else + decoder = LZ4F_decompress_safe; if ((size_t)(dstEnd-dstPtr) < dctxPtr->maxBlockSize) /* not enough room : decode into tmpOut */ { - decodedSize = LZ4_decompress_safe((const char*)selectedIn, (char*)dctxPtr->tmpOut, (int)dctxPtr->sizeToDecode, (int)dctxPtr->maxBlockSize); + decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut, (int)dctxPtr->sizeToDecode, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); if (decodedSize < 0) return -ERROR_GENERIC; /* decompression failed */ if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize); + if (dctxPtr->frameInfo.blockMode==blockLinked) + LZ4F_saveDict(dctxPtr, dctxPtr->tmpOut, decodedSize); dctxPtr->tmpOutSize = decodedSize; dctxPtr->tmpOutStart = 0; dctxPtr->dStage = dstage_flushOut; break; } - decodedSize = LZ4_decompress_safe((const char*)selectedIn, (char*)dstPtr, (int)dctxPtr->sizeToDecode, (int)dctxPtr->maxBlockSize); + decodedSize = decoder((const char*)selectedIn, (char*)dstPtr, (int)dctxPtr->sizeToDecode, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); if (decodedSize < 0) return -ERROR_GENERIC; /* decompression failed */ if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize); + if (dctxPtr->frameInfo.blockMode==blockLinked) + LZ4F_saveDict(dctxPtr, dstPtr, decodedSize); dstPtr += decodedSize; dctxPtr->dStage = dstage_getCBlockSize; break; |