From 9e501702bd1137b4049d06edc60ec366ec725d34 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 9 Apr 2019 18:10:02 -0700 Subject: made LZ4F_getHeaderSize() public --- lib/lz4frame.c | 90 +++++++++++++++++++++++++++++-------------------------- lib/lz4frame.h | 66 ++++++++++++++++++++++++++++++---------- tests/frametest.c | 11 +++++-- 3 files changed, 107 insertions(+), 60 deletions(-) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index efe38da..2b31a0d 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -211,7 +211,7 @@ static void LZ4F_writeLE64 (void* dst, U64 value64) #define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U #define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB -static const size_t minFHSize = 7; +static const size_t minFHSize = LZ4F_HEADER_SIZE_MIN; /* 7 */ static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 19 */ static const size_t BHSize = 4; /* block header : size, and compress flag */ static const size_t BFSize = 4; /* block footer : checksum (optional) */ @@ -280,8 +280,9 @@ size_t LZ4F_getBlockSize(unsigned blockSizeID) static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB }; if (blockSizeID == 0) blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; - if (blockSizeID < 4 || blockSizeID > 7) return err0r(LZ4F_ERROR_maxBlockSize_invalid); - blockSizeID -= 4; + if (blockSizeID < LZ4F_max64KB || blockSizeID > LZ4F_max4MB) + return err0r(LZ4F_ERROR_maxBlockSize_invalid); + blockSizeID -= LZ4F_max64KB; return blockSizes[blockSizeID]; } @@ -1094,31 +1095,6 @@ void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx) } -/*! LZ4F_headerSize() : - * @return : size of frame header - * or an error code, which can be tested using LZ4F_isError() - */ -static size_t LZ4F_headerSize(const void* src, size_t srcSize) -{ - /* minimal srcSize to determine header size */ - if (srcSize < 5) return err0r(LZ4F_ERROR_frameHeader_incomplete); - - /* special case : skippable frames */ - if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) return 8; - - /* control magic number */ - if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER) - return err0r(LZ4F_ERROR_frameType_unknown); - - /* Frame Header Size */ - { BYTE const FLG = ((const BYTE*)src)[4]; - U32 const contentSizeFlag = (FLG>>3) & _1BIT; - U32 const dictIDFlag = FLG & _1BIT; - return minFHSize + (contentSizeFlag*8) + (dictIDFlag*4); - } -} - - /*! LZ4F_decodeHeader() : * input : `src` points at the **beginning of the frame** * output : set internal values of dctx, such as @@ -1191,6 +1167,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize } /* check header */ + assert(frameHeaderSize > 5); { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5); if (HC != srcPtr[frameHeaderSize-1]) return err0r(LZ4F_ERROR_headerChecksum_invalid); @@ -1214,6 +1191,34 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize } +/*! LZ4F_headerSize() : + * @return : size of frame header + * or an error code, which can be tested using LZ4F_isError() + */ +size_t LZ4F_headerSize(const void* src, size_t srcSize) +{ + if (src == NULL) return err0r(LZ4F_ERROR_srcPtr_wrong); + + /* minimal srcSize to determine header size */ + if (srcSize < LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH) + return err0r(LZ4F_ERROR_frameHeader_incomplete); + + /* special case : skippable frames */ + if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) + return 8; + + /* control magic number */ + if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER) + return err0r(LZ4F_ERROR_frameType_unknown); + + /* Frame Header Size */ + { BYTE const FLG = ((const BYTE*)src)[4]; + U32 const contentSizeFlag = (FLG>>3) & _1BIT; + U32 const dictIDFlag = FLG & _1BIT; + return minFHSize + (contentSizeFlag*8) + (dictIDFlag*4); + } +} + /*! LZ4F_getFrameInfo() : * This function extracts frame parameters (max blockSize, frame checksum, etc.). * Usage is optional. Objective is to provide relevant information for allocation purposes. @@ -1229,10 +1234,12 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize * note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped. * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure. */ -LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoPtr, - const void* srcBuffer, size_t* srcSizePtr) +LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, + LZ4F_frameInfo_t* frameInfoPtr, + const void* srcBuffer, size_t* srcSizePtr) { - if (dctx->dStage > dstage_storeFrameHeader) { /* assumption : dstage_* header enum at beginning of range */ + LZ4F_STATIC_ASSERT(dstage_getFrameHeader < dstage_storeFrameHeader); + if (dctx->dStage > dstage_storeFrameHeader) { /* frameInfo already decoded */ size_t o=0, i=0; *srcSizePtr = 0; @@ -1245,7 +1252,6 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoP *srcSizePtr = 0; return err0r(LZ4F_ERROR_frameDecoding_alreadyStarted); } else { - size_t decodeResult; size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr); if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; } if (*srcSizePtr < hSize) { @@ -1253,16 +1259,16 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoP return err0r(LZ4F_ERROR_frameHeader_incomplete); } - decodeResult = LZ4F_decodeHeader(dctx, srcBuffer, hSize); - if (LZ4F_isError(decodeResult)) { - *srcSizePtr = 0; - } else { - *srcSizePtr = decodeResult; - decodeResult = BHSize; /* block header size */ - } - *frameInfoPtr = dctx->frameInfo; - return decodeResult; - } } + { size_t decodeResult = LZ4F_decodeHeader(dctx, srcBuffer, hSize); + if (LZ4F_isError(decodeResult)) { + *srcSizePtr = 0; + } else { + *srcSizePtr = decodeResult; + decodeResult = BHSize; /* block header size */ + } + *frameInfoPtr = dctx->frameInfo; + return decodeResult; + } } } } diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 68f4118..2ada8b8 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -247,7 +247,8 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); /*---- Compression ----*/ -#define LZ4F_HEADER_SIZE_MAX 19 /* LZ4 Frame header size can vary from 7 to 19 bytes */ +#define LZ4F_HEADER_SIZE_MIN 7 /* LZ4 Frame header size can vary, depending on selected paramaters */ +#define LZ4F_HEADER_SIZE_MAX 19 /*! LZ4F_compressBegin() : * will write the frame header into dstBuffer. @@ -352,23 +353,58 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx); * Streaming decompression functions *************************************/ +#define LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH 5 + +/*! LZ4F_headerSize() : v1.9.0+ + * Provide the header size of a frame starting at `src`. + * `srcSize` must be >= LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH, + * which is enough to decode the header length. + * @return : size of frame header + * or an error code, which can be tested using LZ4F_isError() + * note : Frame header size is variable, but is guaranteed to be + * >= LZ4F_HEADER_SIZE_MIN bytes, and <= LZ4F_HEADER_SIZE_MAX bytes. + */ +size_t LZ4F_headerSize(const void* src, size_t srcSize); + /*! LZ4F_getFrameInfo() : * This function extracts frame parameters (max blockSize, dictID, etc.). - * Its usage is optional. - * Extracted information is typically useful for allocation and dictionary. - * This function works in 2 situations : - * - At the beginning of a new frame, in which case - * it will decode information from `srcBuffer`, starting the decoding process. - * Input size must be large enough to successfully decode the entire frame header. - * Frame header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes. - * It's allowed to provide more input data than this minimum. - * - After decoding has been started. - * In which case, no input is read, frame parameters are extracted from dctx. - * - If decoding has barely started, but not yet extracted information from header, + * Its usage is optional: user can call LZ4F_decompress() directly. + * + * Extracted information will fill an existing LZ4F_frameInfo_t structure. + * This can be useful for allocation and dictionary identification purposes. + * + * LZ4F_getFrameInfo() can work in the following situations : + * + * 1) At the beginning of a new frame, before any invocation of LZ4F_decompress(). + * It will decode header from `srcBuffer`, + * consuming the header and starting the decoding process. + * + * Input size must be large enough to contain the full frame header. + * Frame header size can be known beforehand by LZ4F_headerSize(). + * Frame header size is variable, but is guaranteed to be >= LZ4F_HEADER_SIZE_MIN bytes, + * and not more than <= LZ4F_HEADER_SIZE_MAX bytes. + * Hence, blindly providing LZ4F_HEADER_SIZE_MAX bytes or more will always work. + * It's allowed to provide more input data than the header size, + * LZ4F_getFrameInfo() will only consume the header. + * + * If input size is not large enough, + * aka if it's smaller than header size, + * function will fail and return an error code. + * + * 2) After decoding has been started, + * it's possible to invoke LZ4F_getFrameInfo() anytime + * to extract already decoded frame parameters stored within dctx. + * + * Note that, if decoding has barely started, + * and not yet read enough information to decode the header, * LZ4F_getFrameInfo() will fail. - * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). - * Decompression must resume from (srcBuffer + *srcSizePtr). - * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call, + * + * The number of bytes consumed from srcBuffer will be updated in *srcSizePtr (necessarily <= original value). + * LZ4F_getFrameInfo() only consumes bytes when decoding has not yet started, + * and when decoding the header has been successful. + * Decompression must then resume from (srcBuffer + *srcSizePtr). + * + * @return : a hint about how many srcSize bytes LZ4F_decompress() expects for next call, * or an error code which can be tested using LZ4F_isError(). * note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely. * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure. diff --git a/tests/frametest.c b/tests/frametest.c index fa005db..59e866c 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -211,9 +211,13 @@ int basicTests(U32 seed, double compressibility) CHECK ( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) ); DISPLAYLEVEL(3, "LZ4F_getFrameInfo on null-content frame (#157) \n"); - { size_t avail_in = cSize; - LZ4F_frameInfo_t frame_info; + assert(cSize >= LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH); + { LZ4F_frameInfo_t frame_info; + size_t const fhs = LZ4F_headerSize(compressedBuffer, LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH); + size_t avail_in = fhs; + CHECK( fhs ); CHECK( LZ4F_getFrameInfo(dCtx, &frame_info, compressedBuffer, &avail_in) ); + if (avail_in != fhs) goto _output_error; /* must consume all, since header size is supposed to be exact */ } DISPLAYLEVEL(3, "LZ4F_freeDecompressionContext \n"); @@ -306,7 +310,8 @@ int basicTests(U32 seed, double compressibility) } DISPLAYLEVEL(3, "LZ4F_getFrameInfo on enough input : "); - iSize = 15 - iSize; + iSize = LZ4F_headerSize(ip, LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH); + CHECK( iSize ); CHECK( LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize) ); DISPLAYLEVEL(3, " correctly decoded \n"); } -- cgit v0.12