diff options
| -rw-r--r-- | lz4frame.c | 431 | ||||
| -rw-r--r-- | lz4frame.h | 17 | ||||
| -rw-r--r-- | programs/Makefile | 12 | ||||
| -rw-r--r-- | programs/frametest.c | 327 |
4 files changed, 703 insertions, 84 deletions
@@ -36,22 +36,6 @@ * All related operations, including memory management, are handled by the library. * You don't need lz4.h when using lz4frame.h. * */ - - -/************************************** - CPU Feature Detection -**************************************/ -/* 32 or 64 bits ? */ -static const int LZ4F_32bits = (sizeof(void*)==4); -static const int LZ4F_64bits = (sizeof(void*)==8); - -/* Little Endian or Big Endian ? */ -typedef union { - int num; - char bytes[4]; - } endian_t; -static const endian_t endianTest = { .num = 1 }; -#define LZ4F_isLittleEndian (endianTest.bytes[0]) /************************************** @@ -62,6 +46,10 @@ static const endian_t endianTest = { .num = 1 }; #endif #define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +#ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ +# pragma GCC diagnostic ignored "-Wmissing-field-initializers" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ +#endif /************************************** @@ -101,38 +89,6 @@ static const endian_t endianTest = { .num = 1 }; typedef unsigned long long U64; #endif -#if defined(__GNUC__) -# define _PACKED __attribute__ ((packed)) -#else -# define _PACKED -#endif - -#if !defined(__GNUC__) -# if defined(__IBMC__) || defined(__SUNPRO_C) || defined(__SUNPRO_CC) -# pragma pack(1) -# else -# pragma pack(push, 1) -# endif -#endif - -typedef struct { U16 v; } _PACKED U16_S; -typedef struct { U32 v; } _PACKED U32_S; -typedef struct { U64 v; } _PACKED U64_S; -typedef struct {size_t v;} _PACKED size_t_S; - -#if !defined(__GNUC__) -# if defined(__SUNPRO_C) || defined(__SUNPRO_CC) -# pragma pack(0) -# else -# pragma pack(pop) -# endif -#endif - -#define A16(x) (((U16_S *)(x))->v) -#define A32(x) (((U32_S *)(x))->v) -#define A64(x) (((U64_S *)(x))->v) -#define AARCH(x) (((size_t_S *)(x))->v) - /************************************** Constants @@ -141,12 +97,30 @@ typedef struct {size_t v;} _PACKED size_t_S; #define MB *(1<<20) #define GB *(1<<30) +#define _1BIT 0x01 +#define _2BITS 0x03 +#define _3BITS 0x07 +#define _4BITS 0x0F +#define _8BITS 0xFF + +#define LZ4F_MAGICNUMBER 0x184D2204U +#define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U +#define LZ4F_MAXHEADERFRAME_SIZE 19 #define LZ4F_BLOCKSIZEID_DEFAULT 4 /************************************** Structures and local types **************************************/ +typedef struct { + LZ4F_preferences_t prefs; + unsigned version; + unsigned cStage; + size_t maxBlockSize; + XXH32_stateSpace_t xxh; + BYTE* tmpInputBuffer; + size_t tmpInputFilled; +} LZ4F_cctx_internal_t; /************************************** @@ -160,14 +134,32 @@ typedef struct {size_t v;} _PACKED size_t_S; static size_t LZ4F_getBlockSize(unsigned blockSizeID) { static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB }; - + if (blockSizeID == 0) blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; blockSizeID -= 4; - if (blockSizeID > 3) return ERROR_maxBlockSize_invalid; + if (blockSizeID > 3) return -ERROR_maxBlockSize_invalid; return blockSizes[blockSizeID]; } +/* unoptimized version; solves endianess & alignment issues */ +static void LZ4F_writeLE32 (BYTE* dstPtr, U32 value32) +{ + dstPtr[0] = (BYTE)value32; + dstPtr[1] = (BYTE)(value32 >> 8); + dstPtr[2] = (BYTE)(value32 >> 16); + dstPtr[3] = (BYTE)(value32 >> 24); +} + + +static BYTE LZ4F_headerChecksum (BYTE* header, size_t length) +{ + U32 xxh = XXH32(header, length, 0); + return (BYTE)(xxh >> 8); +} + + + /************************************** Error management **************************************/ @@ -178,7 +170,7 @@ int LZ4F_isError(LZ4F_errorCode_t code) /************************************** - Compression functions + Simple compression functions **************************************/ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_frameInfo_t* frameInfoPtr) { @@ -189,22 +181,22 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_frameInfo_t* frameInfo unsigned nbBlocks; size_t frameSuffixSize; size_t totalBound; - + if (frameInfoPtr==NULL) frameInfoPtr = &frameInfoNull; /* all parameters set to default */ - + headerSize = 7; /* basic header size (no option) including magic number */ blockInfoSize = 4; /* basic blockInfo size (no option) for one block */ - - blockSize = LZ4F_getBlockSize(frameInfoPtr->maxBlockSizeID); + + blockSize = LZ4F_getBlockSize(frameInfoPtr->blockSizeID); nbBlocks = (srcSize + (blockSize-1)) / blockSize; blockInfoSize *= nbBlocks; /* total block info size */ - + frameSuffixSize = 4; /* basic frameSuffixSize (no option) */ if (frameInfoPtr->contentChecksumFlag == contentChecksumEnabled) frameSuffixSize += 4; - + totalBound = headerSize + srcSize + blockInfoSize + frameSuffixSize; - if (totalBound < srcSize) return ERROR_srcSize_tooLarge; /* overflow error */ - + if (totalBound < srcSize) return -ERROR_srcSize_tooLarge; /* overflow error */ + return totalBound; } @@ -222,27 +214,27 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf { const LZ4F_frameInfo_t frameInfoNull = { 0 }; const LZ4F_frameInfo_t* const frameInfoPtr = (preferencesPtr==NULL) ? &frameInfoNull : &(preferencesPtr->frameInfo); - LZ4F_compressionContext_t cctx; + LZ4F_compressionContext_t cctx = NULL; LZ4F_errorCode_t errorCode; BYTE* const dstStart = (BYTE*) dstBuffer; BYTE* dstPtr = dstStart; - size_t blockSize = LZ4F_getBlockSize(frameInfoPtr->maxBlockSizeID); + size_t blockSize = LZ4F_getBlockSize(frameInfoPtr->blockSizeID); unsigned nbBlocks = (srcSize + (blockSize-1)) / blockSize; unsigned blockNb; const BYTE* srcPtr = (const BYTE*) srcBuffer; const size_t dstBlockSize = LZ4F_compressBound(blockSize, frameInfoPtr); - + if (dstMaxSize < LZ4F_compressFrameBound(srcSize, frameInfoPtr)) - return ERROR_maxDstSize_tooSmall; - + return -ERROR_dstMaxSize_tooSmall; + errorCode = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION, preferencesPtr); if (LZ4F_isError(errorCode)) return errorCode; - + errorCode = LZ4F_compressBegin(cctx, dstBuffer, dstMaxSize); /* write header */ if (LZ4F_isError(errorCode)) return errorCode; dstPtr += errorCode; /* header size */ - + for (blockNb=1; blockNb<nbBlocks; blockNb++) { errorCode = LZ4F_compress(cctx, dstPtr, dstBlockSize, srcPtr, blockSize, NULL); @@ -250,13 +242,16 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf srcPtr += blockSize; dstPtr += errorCode; } - + /* last block */ - blockSize = srcSize % blockSize; - errorCode = LZ4F_compress(cctx, dstPtr, dstBlockSize, srcPtr, blockSize, NULL); - if (LZ4F_isError(errorCode)) return errorCode; - dstPtr += errorCode; - + { + size_t lastBlockSize = srcSize % blockSize; + if (lastBlockSize==0) lastBlockSize = blockSize; + errorCode = LZ4F_compress(cctx, dstPtr, dstBlockSize, srcPtr, lastBlockSize, NULL); + if (LZ4F_isError(errorCode)) return errorCode; + dstPtr += errorCode; + } + errorCode = LZ4F_compressEnd(cctx, dstPtr, dstMaxSize, NULL); /* flush last block, and generate suffix */ if (LZ4F_isError(errorCode)) return errorCode; dstPtr += errorCode; @@ -266,3 +261,295 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf return (dstPtr - dstStart); } + + +/********************************** + * Advanced compression functions + * *********************************/ + +/* LZ4F_createCompressionContext() : + * The first thing to do is to create a compressionContext object, which will be used in all compression operations. + * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. + * The version provided MUST be LZ4F_VERSION. It is intended to track potential version differences between different binaries. + * The LZ4F_preferences_t structure is optional : you can provide NULL as argument, all preferences will then be set to default. + * The function will provide a pointer to a fully allocated LZ4F_compressionContext_t object. + * If the result LZ4F_errorCode_t is not zero, there was an error during context creation. + * Object can release its memory using LZ4F_freeCompressionContext(); + */ +LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, int version, const LZ4F_preferences_t* preferencesPtr) +{ + const LZ4F_preferences_t prefNull = { 0 }; + LZ4F_cctx_internal_t* cctxPtr; + + if (preferencesPtr == NULL) preferencesPtr = &prefNull; + + cctxPtr = malloc(sizeof(LZ4F_cctx_internal_t)); + if (cctxPtr==NULL) return -ERROR_allocation_failed; + + cctxPtr->prefs = *preferencesPtr; /* equivalent to memcpy() */ + cctxPtr->version = version; + cctxPtr->cStage = 0; /* Next stage : write header */ + if (cctxPtr->prefs.frameInfo.blockSizeID == 0) cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; + cctxPtr->maxBlockSize = LZ4F_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID); + cctxPtr->tmpInputBuffer = malloc(cctxPtr->maxBlockSize); + if (cctxPtr->tmpInputBuffer == NULL) return -ERROR_allocation_failed; + cctxPtr->tmpInputFilled = 0; + XXH32_resetState(&(cctxPtr->xxh), 0); + + *LZ4F_compressionContextPtr = (LZ4F_compressionContext_t)cctxPtr; + + return OK_NoError; +} + + +LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_compressionContext) +{ + LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)LZ4F_compressionContext; + + free(cctxPtr->tmpInputBuffer); + free(LZ4F_compressionContext); + + return OK_NoError; +} + + +/* LZ4F_compressBegin() : + * will write the frame header into dstBuffer. + * dstBuffer must be large enough to accomodate a header (dstMaxSize). Maximum header size is LZ4F_MAXHEADERFRAME_SIZE(19) bytes. + * The result of the function is the number of bytes written into dstBuffer for the header + * or an error code (can be tested using LZ4F_isError()) + */ +size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize) +{ + LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + BYTE* headerStart; + + if (dstMaxSize < LZ4F_MAXHEADERFRAME_SIZE) return -ERROR_dstMaxSize_tooSmall; + if (cctxPtr->cStage != 0) return -ERROR_GENERIC; + + /* Magic Number */ + LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER); + dstPtr += 4; + headerStart = dstPtr; + + /* FLG Byte */ + *dstPtr = (1 & _2BITS) << 6; /* Version('01') */ + *dstPtr |= (1 & _1BIT ) << 5; /* Blocks independents */ + *dstPtr |= (1 & _1BIT ) << 2; /* Stream checksum */ + dstPtr++; + /* BD Byte */ + *dstPtr = (char)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4); + dstPtr++; + /* CRC Byte */ + *dstPtr = LZ4F_headerChecksum(headerStart, 2); + dstPtr++; + + cctxPtr->cStage = 1; /* header written */ + + return (dstPtr - dstStart); +} + + +/* LZ4F_compressBound() : gives the size of Dst buffer given a srcSize to handle worst case situations. + * The LZ4F_frameInfo_t structure is optional : + * you can provide NULL as argument, all preferences will then be set to default. + * */ +size_t LZ4F_compressBound(size_t srcSize, const LZ4F_frameInfo_t* frameInfoPtr) +{ + blockSizeID_t bid = (frameInfoPtr==NULL) ? LZ4F_BLOCKSIZEID_DEFAULT : frameInfoPtr->blockSizeID; + size_t blockSize = LZ4F_getBlockSize(bid); + size_t vSrcSize = srcSize + (blockSize-1); /* worst case : tmp buffer almost filled */ + unsigned nbBlocks = vSrcSize / blockSize; + size_t blockInfo = 4; /* default, without block CRC option */ + + return (blockSize + blockInfo) * nbBlocks; +} + +/* LZ4F_getMaxSrcSize() : gives max allowed srcSize given dstMaxSize to handle worst case situations. + * You can use dstMaxSize==0 to know the "natural" srcSize instead (block size). + * The LZ4F_frameInfo_t structure is optional : + * you can provide NULL as argument, all preferences will then be set to default. + * */ +size_t LZ4F_getMaxSrcSize(size_t dstMaxSize, const LZ4F_frameInfo_t* frameInfoPtr) +{ + blockSizeID_t bid = (frameInfoPtr==NULL) ? LZ4F_BLOCKSIZEID_DEFAULT : frameInfoPtr->blockSizeID; + size_t blockSize = LZ4F_getBlockSize(bid); + size_t worstCBlockSize = blockSize + 4; /* default, with no block CRC option */ + unsigned nbBlocks = dstMaxSize / worstCBlockSize; + size_t maxSrcSize = nbBlocks * blockSize; + + if (dstMaxSize == 0) return blockSize; + if (nbBlocks == 0) return -ERROR_dstMaxSize_tooSmall; /* can't even fit one block */ + + return maxSrcSize; +} + + +/* LZ4F_compress() + * You can then call LZ4F_compress() repetitively to compress as much data as necessary. + * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. + * You can get the minimum value of dstMaxSize by using LZ4F_compressBound() + * Conversely, given a fixed dstMaxSize value, you can know the maximum srcSize authorized using LZ4F_getMaxSrcSize() + * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode) + * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + * The result of the function is the number of bytes written into dstBuffer (it can be zero, meaning input data is just stored within compressionContext for a future block to complete) + * The function outputs an error code if it fails (can be tested using LZ4F_isError()) + */ +size_t LZ4F_compress(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr) +{ + LZ4F_compressOptions_t cOptionsNull = { 0 }; + LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; + size_t blockSize = cctxPtr->maxBlockSize; + const BYTE* srcPtr = (const BYTE*)srcBuffer; + const BYTE* const srcEnd = srcPtr + srcSize; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + + + if (cctxPtr->cStage != 1) return -ERROR_GENERIC; + if (dstMaxSize < LZ4F_compressBound(srcSize, &(cctxPtr->prefs.frameInfo))) return -ERROR_dstMaxSize_tooSmall; + if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; + + /* complete tmp buffer */ + if (cctxPtr->tmpInputFilled > 0) + { + size_t sizeToCopy = blockSize - cctxPtr->tmpInputFilled; + if (sizeToCopy > srcSize) + { + /* add to tmp buffer */ + memcpy(cctxPtr->tmpInputBuffer + cctxPtr->tmpInputFilled, srcBuffer, srcSize); + srcPtr = srcEnd; + cctxPtr->tmpInputFilled += srcSize; + } + else + { + BYTE* cSizePtr = dstPtr; + U32 cSize; + memcpy(cctxPtr->tmpInputBuffer + cctxPtr->tmpInputFilled, srcBuffer, sizeToCopy); + srcPtr += sizeToCopy; + dstPtr += 4; /* space for cSizePtr */ + cSize = (U32)LZ4_compress_limitedOutput((const char*)cctxPtr->tmpInputBuffer, (char*)dstPtr, (int)(blockSize), (int)(blockSize-1)); + dstPtr += cSize; + LZ4F_writeLE32(cSizePtr, cSize); + if (cSize == 0) /* compression failed */ + { + cSize = blockSize + LZ4F_BLOCKUNCOMPRESSED_FLAG; + LZ4F_writeLE32(cSizePtr, cSize); + memcpy(dstPtr, cctxPtr->tmpInputBuffer, blockSize); + dstPtr += blockSize; + } + cctxPtr->tmpInputFilled = 0; + } + } + + while ((size_t)(srcEnd - srcPtr) >= blockSize) + { + /* compress one block */ + BYTE* cSizePtr = dstPtr; + U32 cSize; + dstPtr += 4; /* space for cSizePtr */ + cSize = (U32)LZ4_compress_limitedOutput((const char*)srcPtr, (char*)dstPtr, (int)(blockSize), (int)(blockSize-1)); + dstPtr += cSize; + LZ4F_writeLE32(cSizePtr, cSize); + if (cSize == 0) /* compression failed */ + { + cSize = blockSize + LZ4F_BLOCKUNCOMPRESSED_FLAG; + LZ4F_writeLE32(cSizePtr, cSize); + memcpy(dstPtr, srcPtr, blockSize); + dstPtr += blockSize; + } + srcPtr += blockSize; + } + + if (srcPtr < srcEnd) + { + /* fill tmp buffer */ + size_t sizeToCopy = srcEnd - srcPtr; + memcpy(cctxPtr->tmpInputBuffer, srcPtr, sizeToCopy); + cctxPtr->tmpInputFilled = sizeToCopy; + } + + if (cctxPtr->prefs.frameInfo.contentChecksumFlag == contentChecksumEnabled) + XXH32_update(&(cctxPtr->xxh), srcBuffer, (unsigned)srcSize); + + return dstPtr - dstStart; +} + + +/* LZ4F_flush() + * Should you need to create compressed data immediately, without waiting for a block to be filled, + * you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext. + * The result of the function is the number of bytes written into dstBuffer + * (it can be zero, this means there was no data left within compressionContext) + * The function outputs an error code if it fails (can be tested using LZ4F_isError()) + * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + */ +size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) +{ + LZ4F_compressOptions_t cOptionsNull = { 0 }; + LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + + + if (cctxPtr->tmpInputFilled == 0) return 0; /* nothing to flush */ + if (cctxPtr->cStage != 1) return -ERROR_GENERIC; + if (dstMaxSize < LZ4F_compressBound(1, &(cctxPtr->prefs.frameInfo))) return -ERROR_dstMaxSize_tooSmall; + if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; + + { + BYTE* cSizePtr = dstPtr; + U32 cSize; + dstPtr += 4; /* space for cSizePtr */ + cSize = (U32)LZ4_compress_limitedOutput((const char*)cctxPtr->tmpInputBuffer, (char*)dstPtr, (int)(cctxPtr->tmpInputFilled), (int)(cctxPtr->tmpInputFilled-1)); + dstPtr += cSize; + LZ4F_writeLE32(cSizePtr, cSize); + if (cSize == 0) /* compression failed */ + { + cSize = cctxPtr->tmpInputFilled + LZ4F_BLOCKUNCOMPRESSED_FLAG; + LZ4F_writeLE32(cSizePtr, cSize); + memcpy(dstPtr, cctxPtr->tmpInputBuffer, cctxPtr->tmpInputFilled); + dstPtr += cctxPtr->tmpInputFilled; + } + cctxPtr->tmpInputFilled = 0; + } + + return dstPtr - dstStart; +} + + +/* LZ4F_compressEnd() + * When you want to properly finish the compressed frame, just call LZ4F_compressEnd(). + * It will flush whatever data remained within compressionContext (like LZ4_flush()) + * but also properly finalize the frame, with an endMark and a checksum. + * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) + * The function outputs an error code if it fails (can be tested using LZ4F_isError()) + * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + * compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same. + */ +size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) +{ + LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + size_t errorCode; + + errorCode = LZ4F_flush(compressionContext, dstBuffer, dstMaxSize, compressOptionsPtr); + if (LZ4F_isError(errorCode)) return errorCode; + dstPtr += errorCode; + + LZ4F_writeLE32(dstPtr, 0); dstPtr+=4; /* endMark */ + + if (cctxPtr->prefs.frameInfo.contentChecksumFlag == contentChecksumEnabled) + { + U32 xxh = XXH32_intermediateDigest(&(cctxPtr->xxh)); + LZ4F_writeLE32(dstPtr, xxh); + dstPtr+=4; /* content Checksum */ + } + + cctxPtr->cStage = 0; /* state is now re-usable (with identical preferences) */ + + return dstPtr - dstStart; +} @@ -58,8 +58,9 @@ typedef size_t LZ4F_errorCode_t; typedef enum { OK_FrameEnd = 1 } LZ4F_successCodes; typedef enum { OK_NoError = 0, ERROR_GENERIC = 1, ERROR_maxBlockSize_invalid, ERROR_blockMode_invalid, ERROR_contentChecksumFlag_invalid, + ERROR_srcSize_tooLarge, ERROR_dstMaxSize_tooSmall, + ERROR_allocation_failed, ERROR_compressionLevel_invalid, - ERROR_srcSize_tooLarge, ERROR_maxDstSize_tooSmall, ERROR_maxCode } LZ4F_errorCodes; /* error codes are negative unsigned values. Compare function result to (-specificCode) */ @@ -71,14 +72,14 @@ int LZ4F_isError(LZ4F_errorCode_t code); /* Basically : code > -ERROR_maxCode Framing compression functions **************************************/ -typedef enum { LZ4F_default=0, max64KB=4, max256KB=5, max1MB=6, max4MB=7} maxBlockSize_t; +typedef enum { LZ4F_default=0, max64KB=4, max256KB=5, max1MB=6, max4MB=7} blockSizeID_t; typedef enum { blockLinked=1, blockIndependent} blockMode_t; typedef enum { contentChecksumEnabled, noContentChecksum} contentChecksum_t; typedef struct { - maxBlockSize_t maxBlockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ - blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */ - contentChecksum_t contentChecksumFlag; /* contentChecksumEnabled (default), noContentChecksum ; */ + blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ + blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */ + contentChecksum_t contentChecksumFlag; /* contentChecksumEnabled (default), noContentChecksum ; */ } LZ4F_frameInfo_t; typedef struct { @@ -89,7 +90,7 @@ typedef struct { -/********************************** +/*********************************** * Simple compression function * *********************************/ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_frameInfo_t* frameInfoPtr); @@ -138,7 +139,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize); /* LZ4F_compressBegin() : * will write the frame header into dstBuffer. - * dstBuffer must be large enough to accomodate a header (dstMaxSize). Maximum header size is 15 bytes. + * dstBuffer must be large enough to accomodate a header (dstMaxSize). Maximum header size is 19 bytes. * The result of the function is the number of bytes written into dstBuffer for the header * or an error code (can be tested using LZ4F_isError()) */ @@ -148,7 +149,7 @@ size_t LZ4F_getMaxSrcSize(size_t dstMaxSize, const LZ4F_frameInfo_t* frameInfoPt /* LZ4F_compressBound() : gives the size of Dst buffer given a srcSize to handle worst case situations. * LZ4F_getMaxSrcSize() : gives max allowed srcSize given dstMaxSize to handle worst case situations. * You can use dstMaxSize==0 to know the "natural" srcSize instead (block size). - * The LZ4F_preferences_t structure is optional : you can provide NULL as argument, all preferences will then be set to default. + * The LZ4F_frameInfo_t structure is optional : you can provide NULL as argument, all preferences will then be set to default. */ size_t LZ4F_compress(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr); diff --git a/programs/Makefile b/programs/Makefile index a04c323..441d538 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -30,13 +30,13 @@ # fullbench32: Same as fullbench, but forced to compile in 32-bits mode # ########################################################################## -RELEASE=r122 +RELEASE=r123 DESTDIR?= PREFIX ?= /usr CC := $(CC) CFLAGS ?= -O3 -CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wstrict-prototypes -DLZ4_VERSION=\"$(RELEASE)\" +CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wstrict-prototypes -Wpedantic -DLZ4_VERSION=\"$(RELEASE)\" FLAGS = -I.. $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) BINDIR=$(PREFIX)/bin @@ -68,7 +68,7 @@ endif default: lz4 lz4c -all: lz4 lz4c lz4c32 fullbench fullbench32 fuzzer fuzzer32 datagen +all: lz4 lz4c lz4c32 fullbench fullbench32 fuzzer fuzzer32 frametest datagen lz4: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4io.c lz4cli.c $(CC) $(FLAGS) -DDISABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT) @@ -91,6 +91,9 @@ fuzzer : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fuzzer.c fuzzer32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fuzzer.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) +frametest: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c frametest.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + datagen : datagen.c $(CC) $(FLAGS) $^ -o $@$(EXT) @@ -99,7 +102,8 @@ clean: @rm -f core *.o \ lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) \ fullbench$(EXT) fullbench32$(EXT) \ - fuzzer$(EXT) fuzzer32$(EXT) datagen$(EXT) + fuzzer$(EXT) fuzzer32$(EXT) \ + frametest$(EXT) datagen$(EXT) @echo Cleaning completed diff --git a/programs/frametest.c b/programs/frametest.c new file mode 100644 index 0000000..862ecf4 --- /dev/null +++ b/programs/frametest.c @@ -0,0 +1,327 @@ +/* + frameTest - test tool for lz4frame + Copyright (C) Yann Collet 2014 + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +/************************************** + Compiler specific +**************************************/ +#define _CRT_SECURE_NO_WARNINGS // fgets +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ +#endif + + +/************************************** + Includes +**************************************/ +#include <stdlib.h> +#include <stdio.h> // fgets, sscanf +#include <sys/timeb.h> // timeb +#include <string.h> // strcmp +#include "lz4frame.h" + + +/************************************** + Basic Types +**************************************/ +#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ +# include <stdint.h> + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif + + +/************************************** + Constants +**************************************/ +#ifndef LZ4_VERSION +# define LZ4_VERSION "" +#endif + +#define NB_ATTEMPTS (1<<16) +#define COMPRESSIBLE_NOISE_LENGTH (1 << 21) +#define FUZ_MAX_BLOCK_SIZE (1 << 17) +#define FUZ_MAX_DICT_SIZE (1 << 15) +#define FUZ_COMPRESSIBILITY_DEFAULT 50 +#define PRIME1 2654435761U +#define PRIME2 2246822519U +#define PRIME3 3266489917U + +#define KB *(1U<<10) +#define MB *(1U<<20) +#define GB *(1U<<30) + + +/************************************** + Macros +**************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } + + +/***************************************** + Local Parameters +*****************************************/ +static int no_prompt = 0; +static char* programName; +static int displayLevel = 2; + + +/********************************************************* + Fuzzer functions +*********************************************************/ +static int FUZ_GetMilliStart(void) +{ + struct timeb tb; + int nCount; + ftime( &tb ); + nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000); + return nCount; +} + + +static int FUZ_GetMilliSpan( int nTimeStart ) +{ + int nSpan = FUZ_GetMilliStart() - nTimeStart; + if ( nSpan < 0 ) + nSpan += 0x100000 * 1000; + return nSpan; +} + + +# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) +unsigned int FUZ_rand(unsigned int* src) +{ + U32 rand32 = *src; + rand32 *= PRIME1; + rand32 += PRIME2; + rand32 = FUZ_rotl32(rand32, 13); + *src = rand32; + return rand32 >> 3; +} + + +#define FUZ_RAND15BITS ((FUZ_rand(seed) >> 3) & 32767) +#define FUZ_RANDLENGTH ( ((FUZ_rand(seed) >> 7) & 3) ? (FUZ_rand(seed) % 14) : (FUZ_rand(seed) & 511) + 15) +void FUZ_fillCompressibleNoiseBuffer(void* buffer, int bufferSize, double proba, U32* seed) +{ + BYTE* BBuffer = (BYTE*)buffer; + int pos = 0; + U32 P32 = (U32)(32768 * proba); + + // First Byte + BBuffer[pos++] = (BYTE)(FUZ_rand(seed)); + + while (pos < bufferSize) + { + // Select : Literal (noise) or copy (within 64K) + if (FUZ_RAND15BITS < P32) + { + // Copy (within 64K) + int ref, d; + int length = FUZ_RANDLENGTH + 4; + int offset = FUZ_RAND15BITS + 1; + if (offset > pos) offset = pos; + if (pos + length > bufferSize) length = bufferSize - pos; + ref = pos - offset; + d = pos + length; + while (pos < d) BBuffer[pos++] = BBuffer[ref++]; + } + else + { + // Literal (noise) + int d; + int length = FUZ_RANDLENGTH; + if (pos + length > bufferSize) length = bufferSize - pos; + d = pos + length; + while (pos < d) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5); + } + } +} + + + +#define FUZ_MAX(a,b) (a>b?a:b) + +int frameTest(U32 seed, int nbCycles, int startCycle, double compressibility) +{ + int testResult = 0; + void* CNBuffer; + void* compressedBuffer; + void* decodedBuffer; + U32 randState = seed; + size_t cSize; + + (void)nbCycles; (void)startCycle; + // Create compressible test buffer + CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); + FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); + compressedBuffer = malloc(LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL)); + decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); + + // Trivial test : one-step frame, all default + cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(64 KB, NULL), CNBuffer, 64 KB, NULL); + if (LZ4F_isError(cSize)) goto _output_error; + DISPLAY("Compressed %i bytes into a %i bytes frame \n", 64 KB, (int)cSize); + +_end: + free(CNBuffer); + free(compressedBuffer); + free(decodedBuffer); + return testResult; + +_output_error: + testResult = 1; + if(!no_prompt) getchar(); + goto _end; +} + + +int FUZ_usage(void) +{ + DISPLAY( "Usage :\n"); + DISPLAY( " %s [args]\n", programName); + DISPLAY( "\n"); + DISPLAY( "Arguments :\n"); + DISPLAY( " -i# : Nb of tests (default:%i) \n", NB_ATTEMPTS); + DISPLAY( " -s# : Select seed (default:prompt user)\n"); + DISPLAY( " -t# : Select starting test number (default:0)\n"); + DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); + DISPLAY( " -v : verbose\n"); + DISPLAY( " -h : display help and exit\n"); + return 0; +} + + +int main(int argc, char** argv) { + U32 timestamp = FUZ_GetMilliStart(); + U32 seed=0; + int seedset=0; + int argNb; + int nbTests = NB_ATTEMPTS; + int testNb = 0; + int proba = FUZ_COMPRESSIBILITY_DEFAULT; + + // Check command line + programName = argv[0]; + for(argNb=1; argNb<argc; argNb++) + { + char* argument = argv[argNb]; + + if(!argument) continue; // Protection if argument empty + + // Decode command (note : aggregated commands are allowed) + if (argument[0]=='-') + { + if (!strcmp(argument, "--no-prompt")) { no_prompt=1; seedset=1; displayLevel=1; continue; } + + while (argument[1]!=0) + { + argument++; + switch(*argument) + { + case 'h': + return FUZ_usage(); + case 'v': + argument++; + displayLevel=4; + break; + case 'i': + argument++; + nbTests=0; + while ((*argument>='0') && (*argument<='9')) + { + nbTests *= 10; + nbTests += *argument - '0'; + argument++; + } + break; + case 's': + argument++; + seed=0; seedset=1; + while ((*argument>='0') && (*argument<='9')) + { + seed *= 10; + seed += *argument - '0'; + argument++; + } + break; + case 't': + argument++; + testNb=0; + while ((*argument>='0') && (*argument<='9')) + { + testNb *= 10; + testNb += *argument - '0'; + argument++; + } + break; + case 'p': + argument++; + proba=0; + while ((*argument>='0') && (*argument<='9')) + { + proba *= 10; + proba += *argument - '0'; + argument++; + } + if (proba<0) proba=0; + if (proba>100) proba=100; + break; + default: ; + } + } + } + } + + // Get Seed + printf("Starting lz4frame tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION); + + if (!seedset) + { + char userInput[50] = {0}; + printf("Select an Initialisation number (default : random) : "); + fflush(stdout); + if ( no_prompt || fgets(userInput, sizeof userInput, stdin) ) + { + if ( sscanf(userInput, "%u", &seed) == 1 ) {} + else seed = FUZ_GetMilliSpan(timestamp); + } + } + printf("Seed = %u\n", seed); + if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba); + + if (nbTests<=0) nbTests=1; + + return frameTest(seed, nbTests, testNb, ((double)proba) / 100); +} |