From 7ee725645b58c201f11443d70210c2f986371b59 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 18 Mar 2015 21:38:27 +0100 Subject: frame content size support --- NEWS | 1 + lib/lz4frame.c | 197 +++++++++++++++++++++++++++++++++----------------- lib/lz4frame.h | 17 +++-- lib/lz4frame_static.h | 2 + programs/frametest.c | 76 ++++++++++++++++++- programs/lz4io.c | 2 +- 6 files changed, 217 insertions(+), 78 deletions(-) diff --git a/NEWS b/NEWS index 1901172..8dd7c9b 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,7 @@ r128: New : lz4 cli sparse file support New : command -m, to compress multiple files in a single command +New : lz4frame supports frame content size Fixed : Restored lz4hc compression ratio (was slightly lower since r124) New : lz4frame supports skippable frames Changed:Default "make install" directory is /usr/local diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 47fd84b..644f3e8 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -33,7 +33,7 @@ You can contact the author at : */ /* LZ4F is a stand-alone API to create LZ4-compressed Frames -* in full conformance with specification v1.4.1. +* in full conformance with specification v1.5.0 * All related operations, including memory management, are handled by the library. * */ @@ -100,9 +100,10 @@ typedef unsigned long long U64; #define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U #define LZ4F_MAGICNUMBER 0x184D2204U #define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U -#define LZ4F_MAXHEADERFRAME_SIZE 7 +#define LZ4F_MAXHEADERFRAME_SIZE 15 #define LZ4F_BLOCKSIZEID_DEFAULT max64KB +static const size_t minFHSize = 5; static const U32 minHClevel = 3; /************************************** @@ -111,23 +112,24 @@ static const U32 minHClevel = 3; typedef struct { LZ4F_preferences_t prefs; - U32 version; - U32 cStage; + U32 version; + U32 cStage; size_t maxBlockSize; size_t maxBufferSize; BYTE* tmpBuff; BYTE* tmpIn; size_t tmpInSize; + U64 totalInSize; XXH32_state_t xxh; - void* lz4CtxPtr; - U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */ + void* lz4CtxPtr; + U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */ } LZ4F_cctx_internal_t; typedef struct { LZ4F_frameInfo_t frameInfo; - unsigned version; - unsigned dStage; + U32 version; + U32 dStage; size_t maxBlockSize; size_t maxBufferSize; const BYTE* srcExpect; @@ -141,7 +143,7 @@ typedef struct size_t tmpOutSize; size_t tmpOutStart; XXH32_state_t xxh; - BYTE header[8]; + BYTE header[16]; } LZ4F_dctx_internal_t; @@ -152,7 +154,7 @@ typedef struct static const char* LZ4F_errorStrings[] = { LZ4F_LIST_ERRORS(LZ4F_GENERATE_STRING) }; -U32 LZ4F_isError(LZ4F_errorCode_t code) +unsigned LZ4F_isError(LZ4F_errorCode_t code) { return (code > (LZ4F_errorCode_t)(-ERROR_maxCode)); } @@ -180,6 +182,15 @@ static size_t LZ4F_getBlockSize(unsigned blockSizeID) /* unoptimized version; solves endianess & alignment issues */ +static U32 LZ4F_readLE32 (const BYTE* srcPtr) +{ + U32 value32 = srcPtr[0]; + value32 += (srcPtr[1]<<8); + value32 += (srcPtr[2]<<16); + value32 += (srcPtr[3]<<24); + return value32; +} + static void LZ4F_writeLE32 (BYTE* dstPtr, U32 value32) { dstPtr[0] = (BYTE)value32; @@ -188,19 +199,35 @@ static void LZ4F_writeLE32 (BYTE* dstPtr, U32 value32) dstPtr[3] = (BYTE)(value32 >> 24); } -static U32 LZ4F_readLE32 (const BYTE* srcPtr) +static U64 LZ4F_readLE64 (const BYTE* srcPtr) { - U32 value32 = srcPtr[0]; - value32 += (srcPtr[1]<<8); - value32 += (srcPtr[2]<<16); - value32 += (srcPtr[3]<<24); - return value32; + U64 value64 = srcPtr[0]; + value64 += (srcPtr[1]<<8); + value64 += (srcPtr[2]<<16); + value64 += (srcPtr[3]<<24); + value64 += ((U64)srcPtr[4]<<32); + value64 += ((U64)srcPtr[5]<<40); + value64 += ((U64)srcPtr[6]<<48); + value64 += ((U64)srcPtr[7]<<56); + return value64; +} + +static void LZ4F_writeLE64 (BYTE* dstPtr, U64 value64) +{ + dstPtr[0] = (BYTE)value64; + dstPtr[1] = (BYTE)(value64 >> 8); + dstPtr[2] = (BYTE)(value64 >> 16); + dstPtr[3] = (BYTE)(value64 >> 24); + dstPtr[4] = (BYTE)(value64 >> 32); + dstPtr[5] = (BYTE)(value64 >> 40); + dstPtr[6] = (BYTE)(value64 >> 48); + dstPtr[7] = (BYTE)(value64 >> 56); } -static BYTE LZ4F_headerChecksum (const BYTE* header, size_t length) +static BYTE LZ4F_headerChecksum (const void* header, size_t length) { - U32 xxh = XXH32(header, (U32)length, 0); + U32 xxh = XXH32(header, length, 0); return (BYTE)(xxh >> 8); } @@ -235,7 +262,7 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); prefs.autoFlush = 1; - headerSize = 7; /* basic header size (no option) including magic number */ + headerSize = 15; /* header size, including magic number and frame content size*/ streamSize = LZ4F_compressBound(srcSize, &prefs); return headerSize + streamSize; @@ -269,7 +296,13 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent allocation; works because autoflush==1 & stableSrc==1 */ if (preferencesPtr!=NULL) prefs = *preferencesPtr; - else memset(&prefs, 0, sizeof(prefs)); + else + { + memset(&prefs, 0, sizeof(prefs)); + prefs.frameInfo.frameOSize = (U64)srcSize; + } + if (prefs.frameInfo.frameOSize != 0) + prefs.frameInfo.frameOSize = (U64)srcSize; /* correct frame size if selected (!=0) */ if (prefs.compressionLevel < minHClevel) { @@ -412,13 +445,22 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds /* FLG Byte */ *dstPtr++ = ((1 & _2BITS) << 6) /* Version('01') */ + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) /* Block mode */ - + (char)((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2); /* Stream checksum */ + + (BYTE)((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) /* Frame checksum */ + + (BYTE)((cctxPtr->prefs.frameInfo.frameOSize > 0) << 3); /* Frame content size */ /* BD Byte */ - *dstPtr++ = (char)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4); + *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4); + /* Optional Frame content size field */ + if (cctxPtr->prefs.frameInfo.frameOSize) + { + LZ4F_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.frameOSize); + dstPtr += 8; + cctxPtr->totalInSize = 0; + } /* CRC Byte */ - *dstPtr++ = LZ4F_headerChecksum(headerStart, 2); + *dstPtr = LZ4F_headerChecksum(headerStart, dstPtr - headerStart); + dstPtr++; - cctxPtr->cStage = 1; /* header written, wait for data block */ + cctxPtr->cStage = 1; /* header written, now request input data block */ return (dstPtr - dstStart); } @@ -608,8 +650,9 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d } if (cctxPtr->prefs.frameInfo.contentChecksumFlag == contentChecksumEnabled) - XXH32_update(&(cctxPtr->xxh), srcBuffer, (unsigned)srcSize); + XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize); + cctxPtr->totalInSize += srcSize; return dstPtr - dstStart; } @@ -686,6 +729,12 @@ size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstB cctxPtr->cStage = 0; /* state is now re-usable (with identical preferences) */ + if (cctxPtr->prefs.frameInfo.frameOSize) + { + if (cctxPtr->prefs.frameInfo.frameOSize != cctxPtr->totalInSize) + return (size_t)-ERROR_frameSize_wrong; + } + return dstPtr - dstStart; } @@ -735,7 +784,7 @@ typedef enum { dstage_getHeader=0, dstage_storeHeader, dstage_getCBlock, dstage_storeCBlock, dstage_decodeCBlock, dstage_decodeCBlock_intoDst, dstage_decodeCBlock_intoTmp, dstage_flushOut, dstage_getSuffix, dstage_storeSuffix, - dstage_getSBlockSize, dstage_storeSBlockSize, + dstage_getSFrameSize, dstage_storeSFrameSize, dstage_skipSkippable } dStage_t; @@ -745,72 +794,84 @@ typedef enum { dstage_getHeader=0, dstage_storeHeader, or an error code (testable with LZ4F_isError()) output : set internal values of dctx, such as dctxPtr->frameInfo and dctxPtr->dStage. - */ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const void* srcVoidPtr, size_t srcSize) { BYTE FLG, BD, HC; - unsigned version, blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, dictFlag, blockSizeID; + unsigned version, blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, blockSizeID; size_t bufferNeeded; + size_t frameHeaderSize; const BYTE* srcPtr = (const BYTE*)srcVoidPtr; /* need to decode header to get frameInfo */ - if (srcSize < 7) return (size_t)-ERROR_GENERIC; /* minimal header size */ + if (srcSize < minFHSize) return (size_t)-ERROR_GENERIC; /* minimal header size */ + memset(&(dctxPtr->frameInfo), 0, sizeof(dctxPtr->frameInfo)); /* skippable frames */ if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) { - memset(&(dctxPtr->frameInfo), 0, sizeof(dctxPtr->frameInfo)); dctxPtr->frameInfo.frameType = skippableFrame; if (srcVoidPtr == (void*)(dctxPtr->header)) { - memcpy(dctxPtr->header, dctxPtr->header + 4, srcSize-4); - dctxPtr->tmpInSize = srcSize-4; - dctxPtr->dStage = dstage_storeSBlockSize; + dctxPtr->tmpInSize = srcSize; + dctxPtr->tmpInTarget = 8; + dctxPtr->dStage = dstage_storeSFrameSize; return srcSize; } else { - dctxPtr->dStage = dstage_getSBlockSize; + dctxPtr->dStage = dstage_getSFrameSize; return 4; } } /* control magic number */ - if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) return (size_t)-ERROR_GENERIC; - srcPtr += 4; + if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) return (size_t)-ERROR_frameType_unknown; dctxPtr->frameInfo.frameType = LZ4F_frame; /* Flags */ - FLG = srcPtr[0]; + FLG = srcPtr[4]; version = (FLG>>6) & _2BITS; blockMode = (FLG>>5) & _1BIT; blockChecksumFlag = (FLG>>4) & _1BIT; contentSizeFlag = (FLG>>3) & _1BIT; contentChecksumFlag = (FLG>>2) & _1BIT; - dictFlag = (FLG>>0) & _1BIT; - BD = srcPtr[1]; - blockSizeID = (BD>>4) & _3BITS; - /* check */ - HC = LZ4F_headerChecksum(srcPtr, 2); - if (HC != srcPtr[2]) return (size_t)-ERROR_GENERIC; /* Bad header checksum error */ + /* Frame Header Size */ + frameHeaderSize = contentSizeFlag ? 15 : 7; + + if (srcSize < frameHeaderSize) + { + if (srcPtr != dctxPtr->header) + memcpy(dctxPtr->header, srcPtr, srcSize); + dctxPtr->tmpInSize = srcSize; + dctxPtr->tmpInTarget = frameHeaderSize; + dctxPtr->dStage = dstage_storeHeader; + return srcSize; + } + + BD = srcPtr[5]; + blockSizeID = (BD>>4) & _3BITS; /* validate */ if (version != 1) return (size_t)-ERROR_GENERIC; /* Version Number, only supported value */ if (blockChecksumFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */ - if (contentSizeFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */ - if (((FLG>>1)&_1BIT) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bit */ - if (dictFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */ + if (((FLG>>0)&_2BITS) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bits */ if (((BD>>7)&_1BIT) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bit */ if (blockSizeID < 4) return (size_t)-ERROR_GENERIC; /* 4-7 only supported values for the time being */ if (((BD>>0)&_4BITS) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bits */ + /* check */ + HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5); + if (HC != srcPtr[frameHeaderSize-1]) return (size_t)-ERROR_GENERIC; /* Bad header checksum error */ + /* save */ dctxPtr->frameInfo.blockMode = (blockMode_t)blockMode; dctxPtr->frameInfo.contentChecksumFlag = (contentChecksum_t)contentChecksumFlag; dctxPtr->frameInfo.blockSizeID = (blockSizeID_t)blockSizeID; dctxPtr->maxBlockSize = LZ4F_getBlockSize(blockSizeID); + if (contentSizeFlag) + dctxPtr->frameInfo.frameOSize = LZ4F_readLE64(srcPtr+6); /* init */ if (contentChecksumFlag) XXH32_reset(&(dctxPtr->xxh), 0); @@ -837,7 +898,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const void* srcVo dctxPtr->dStage = dstage_getCBlockSize; - return 7; + return frameHeaderSize; } @@ -872,15 +933,14 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t decompressionCont } +/* redirector, with common prototype */ static int LZ4F_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize) { - (void)dictStart; - (void)dictSize; + (void)dictStart; (void)dictSize; return LZ4_decompress_safe (source, dest, compressedSize, maxDecompressedSize); } - static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp) { if (dctxPtr->dictSize==0) @@ -1011,23 +1071,24 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, break; } dctxPtr->tmpInSize = 0; + dctxPtr->tmpInTarget = 7; dctxPtr->dStage = dstage_storeHeader; } case dstage_storeHeader: { - size_t sizeToCopy = 7 - dctxPtr->tmpInSize; + size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy); dctxPtr->tmpInSize += sizeToCopy; srcPtr += sizeToCopy; - if (dctxPtr->tmpInSize < 7) + if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) { - nextSrcSizeHint = (7 - dctxPtr->tmpInSize) + 4; - doAnotherStage = 0; /* not enough src data, wait to get some more */ + nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + 4; + doAnotherStage = 0; /* not enough src data, ask for some more */ break; } - LZ4F_errorCode_t errorCode = LZ4F_decodeHeader(dctxPtr, dctxPtr->header, 7); + LZ4F_errorCode_t errorCode = LZ4F_decodeHeader(dctxPtr, dctxPtr->header, dctxPtr->tmpInTarget); if (LZ4F_isError(errorCode)) return errorCode; break; } @@ -1292,7 +1353,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, break; } - case dstage_getSBlockSize: + case dstage_getSFrameSize: { if ((srcEnd - srcPtr) >= 4) { @@ -1302,32 +1363,34 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, else { /* not enough input to read cBlockSize field */ - dctxPtr->tmpInSize = 0; - dctxPtr->dStage = dstage_storeSBlockSize; + dctxPtr->tmpInSize = 4; + dctxPtr->tmpInTarget = 8; + dctxPtr->dStage = dstage_storeSFrameSize; } } - if (dctxPtr->dStage == dstage_storeSBlockSize) - case dstage_storeSBlockSize: + if (dctxPtr->dStage == dstage_storeSFrameSize) + case dstage_storeSFrameSize: { - size_t sizeToCopy = 4 - dctxPtr->tmpInSize; + size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy); srcPtr += sizeToCopy; dctxPtr->tmpInSize += sizeToCopy; - if (dctxPtr->tmpInSize < 4) /* not enough input to get full sBlockSize; wait for more */ + if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) /* not enough input to get full sBlockSize; wait for more */ { - nextSrcSizeHint = 4 - dctxPtr->tmpInSize; - doAnotherStage=0; + nextSrcSizeHint = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; + doAnotherStage = 0; break; } - selectedIn = dctxPtr->header; + selectedIn = dctxPtr->header + 4; } /* case dstage_decodeSBlockSize: */ /* no direct access */ { - size_t nextSBlockSize = LZ4F_readLE32(selectedIn); - dctxPtr->tmpInTarget = nextSBlockSize; + size_t SFrameSize = LZ4F_readLE32(selectedIn); + dctxPtr->frameInfo.frameOSize = SFrameSize; + dctxPtr->tmpInTarget = SFrameSize; dctxPtr->dStage = dstage_skipSkippable; break; } diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 71814b0..54db165 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -68,18 +68,19 @@ typedef enum { noContentChecksum=0, contentChecksumEnabled } contentChecksum_t; typedef enum { LZ4F_frame=0, skippableFrame } frameType_t; typedef struct { - blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ - blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */ - contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */ - frameType_t frameType; /* LZ4F_frame, skippableFrame : 0 == default */ - unsigned reserved[4]; + blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ + blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */ + contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */ + frameType_t frameType; /* LZ4F_frame, skippableFrame ; 0 == default */ + unsigned long long frameOSize; /* Size of uncompressed (original) content ; 0 == unknown */ + unsigned reserved[2]; /* must be zero for forward compatibility */ } LZ4F_frameInfo_t; typedef struct { LZ4F_frameInfo_t frameInfo; unsigned compressionLevel; /* 0 == default (fast mode); values above 16 count as 16 */ - unsigned autoFlush; /* 1 == always flush : reduce need for tmp buffer */ - unsigned reserved[4]; + unsigned autoFlush; /* 1 == always flush (reduce need for tmp buffer) */ + unsigned reserved[4]; /* must be zero for forward compatibility */ } LZ4F_preferences_t; @@ -131,7 +132,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_preferences_t* preferencesPtr); /* LZ4F_compressBegin() : * will write the frame header into dstBuffer. - * dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is 19 bytes. + * dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is 15 bytes. * The LZ4F_preferences_t structure is optional : you can provide NULL as argument, all preferences will then be set to default. * The result of the function is the number of bytes written into dstBuffer for the header * or an error code (can be tested using LZ4F_isError()) diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h index db9d167..2e56400 100644 --- a/lib/lz4frame_static.h +++ b/lib/lz4frame_static.h @@ -52,6 +52,8 @@ extern "C" { ITEM(ERROR_compressionLevel_invalid) \ ITEM(ERROR_allocation_failed) \ ITEM(ERROR_srcSize_tooLarge) ITEM(ERROR_dstMaxSize_tooSmall) \ + ITEM(ERROR_frameSize_wrong) \ + ITEM(ERROR_frameType_unknown) \ ITEM(ERROR_wrongSrcPtr) \ ITEM(ERROR_decompressionFailed) \ ITEM(ERROR_checksum_invalid) \ diff --git a/programs/frametest.c b/programs/frametest.c index 62b6fa4..96ac3f9 100644 --- a/programs/frametest.c +++ b/programs/frametest.c @@ -381,6 +381,57 @@ int basicTests(U32 seed, double compressibility) if (LZ4F_isError(cSize)) goto _output_error; DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); + { + size_t errorCode; + BYTE* const ostart = (BYTE*)compressedBuffer; + BYTE* op = ostart; + LZ4F_compressionContext_t cctx; + errorCode = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION); + if (LZ4F_isError(errorCode)) goto _output_error; + + DISPLAYLEVEL(3, "compress without frameSize : \n"); + memset(&(prefs.frameInfo), 0, sizeof(prefs.frameInfo)); + errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs); + if (LZ4F_isError(errorCode)) goto _output_error; + op += errorCode; + errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL); + if (LZ4F_isError(errorCode)) goto _output_error; + op += errorCode; + errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL); + if (LZ4F_isError(errorCode)) goto _output_error; + DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart)); + + DISPLAYLEVEL(3, "compress with frameSize : \n"); + prefs.frameInfo.frameOSize = testSize; + op = ostart; + errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs); + if (LZ4F_isError(errorCode)) goto _output_error; + op += errorCode; + errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL); + if (LZ4F_isError(errorCode)) goto _output_error; + op += errorCode; + errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL); + if (LZ4F_isError(errorCode)) goto _output_error; + DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart)); + + DISPLAYLEVEL(3, "compress with wrong frameSize : \n"); + prefs.frameInfo.frameOSize = testSize+1; + op = ostart; + errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs); + if (LZ4F_isError(errorCode)) goto _output_error; + op += errorCode; + errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL); + if (LZ4F_isError(errorCode)) goto _output_error; + op += errorCode; + errorCode = LZ4F_compressEnd(cctx, op, testSize, NULL); + if (LZ4F_isError(errorCode)) { DISPLAYLEVEL(3, "Error correctly detected : %s \n", LZ4F_getErrorName(errorCode)); } + else + goto _output_error; + + errorCode = LZ4F_freeCompressionContext(cctx); + if (LZ4F_isError(errorCode)) goto _output_error; + } + DISPLAYLEVEL(3, "Skippable frame test : \n"); { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; @@ -412,11 +463,13 @@ int basicTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)decodedBufferSize); /* generate zero-size skippable frame */ + DISPLAYLEVEL(3, "zero-size skippable frame\n"); + ip = (BYTE*)compressedBuffer; + op = (BYTE*)decodedBuffer; FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+1); FUZ_writeLE32(ip+4, 0); iend = ip+8; - DISPLAYLEVEL(3, "random segment sizes : \n"); while (ip < iend) { unsigned nbBits = FUZ_rand(&randState) % maxBits; @@ -428,8 +481,27 @@ int basicTests(U32 seed, double compressibility) op += oSize; ip += iSize; } - DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)decodedBufferSize); + DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8)); + + DISPLAYLEVEL(3, "Skippable frame header complete in first call \n"); + ip = (BYTE*)compressedBuffer; + op = (BYTE*)decodedBuffer; + FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+2); + FUZ_writeLE32(ip+4, 10); + iend = ip+18; + while (ip < iend) + { + size_t iSize = 10; + size_t oSize = 10; + if (iSize > (size_t)(iend-ip)) iSize = iend-ip; + errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); + if (LZ4F_isError(errorCode)) goto _output_error; + op += oSize; + ip += iSize; + } + DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8)); + /* release memory */ errorCode = LZ4F_freeDecompressionContext(dCtx); if (LZ4F_isError(errorCode)) goto _output_error; } diff --git a/programs/lz4io.c b/programs/lz4io.c index 6d36b39..8d356c1 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -625,7 +625,7 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) if (seekResult != 0) EXM_THROW(68, "1 GB skip error (sparse file)"); storedSkips -= 1 GB; } - if (nb0T != seg0SizeT) + if (nb0T != seg0SizeT) /* not all 0s */ { seekResult = fseek(foutput, storedSkips, SEEK_CUR); if (seekResult) EXM_THROW(68, "Skip error (sparse file)"); -- cgit v0.12