diff options
| -rw-r--r-- | lz4frame.c | 265 | ||||
| -rw-r--r-- | lz4frame.h | 42 |
2 files changed, 289 insertions, 18 deletions
diff --git a/lz4frame.c b/lz4frame.c new file mode 100644 index 0000000..8af9c28 --- /dev/null +++ b/lz4frame.c @@ -0,0 +1,265 @@ +/* + LZ4 auto-framing library + Copyright (C) 2011-2014, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +/* LZ4F is a stand-alone API to create LZ4-compressed frames + * fully conformant to specification v1.4.1. + * All related operations, including memory management, are handled by the library. + * You don't need lz4.h when using lz4frame.h. + * */ + + +/************************************** + CPU Feature Detection +**************************************/ +/* 32 or 64 bits ? */ +static const int LZ4F_32bits = (sizeof(void*)==4); +static const int LZ4F_64bits = (sizeof(void*)==8); + +/* Little Endian or Big Endian ? */ +typedef union { + int num; + char bytes[4]; + } endian_t; +static const endian_t endianTest = { .num = 1 }; +#define LZ4F_isLittleEndian (endianTest.bytes[0]) + + +/************************************** + Compiler Options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + + +/************************************** + Memory routines +**************************************/ +#include <stdlib.h> /* malloc, calloc, free */ +#define ALLOCATOR(n,s) calloc(n,s) +#define FREEMEM free +#include <string.h> /* memset, memcpy */ +#define MEM_INIT memset + + +/************************************** + Includes +**************************************/ +#include "lz4frame.h" +#include "lz4.h" +#include "lz4hc.h" +#include "xxhash.h" + + +/************************************** + Basic Types +**************************************/ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ +# include <stdint.h> + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif + +#if defined(__GNUC__) +# define _PACKED __attribute__ ((packed)) +#else +# define _PACKED +#endif + +#if !defined(__GNUC__) +# if defined(__IBMC__) || defined(__SUNPRO_C) || defined(__SUNPRO_CC) +# pragma pack(1) +# else +# pragma pack(push, 1) +# endif +#endif + +typedef struct { U16 v; } _PACKED U16_S; +typedef struct { U32 v; } _PACKED U32_S; +typedef struct { U64 v; } _PACKED U64_S; +typedef struct {size_t v;} _PACKED size_t_S; + +#if !defined(__GNUC__) +# if defined(__SUNPRO_C) || defined(__SUNPRO_CC) +# pragma pack(0) +# else +# pragma pack(pop) +# endif +#endif + +#define A16(x) (((U16_S *)(x))->v) +#define A32(x) (((U32_S *)(x))->v) +#define A64(x) (((U64_S *)(x))->v) +#define AARCH(x) (((size_t_S *)(x))->v) + + +/************************************** + Constants +**************************************/ +#define KB *(1<<10) +#define MB *(1<<20) +#define GB *(1<<30) + + +/************************************** + Structures and local types +**************************************/ + + +/************************************** + Macros +**************************************/ + + +/************************************** + Private functions +**************************************/ +static size_t LZ4F_getBlockSize(unsigned blockSizeID) +{ + static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB }; + + blockSizeID -= 4; + if (blockSizeID > 3) return ERROR_maxBlockSize_invalid; + return blockSizes[blockSizeID]; +} + + +/************************************** + Error management +**************************************/ +int LZ4F_isError(LZ4F_errorCode_t code) +{ + return (code > (LZ4F_errorCode_t)(-ERROR_maxCode)); +} + + +/************************************** + Compression functions +**************************************/ +size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_frameInfo_t* frameInfoPtr) +{ + const LZ4F_frameInfo_t frameInfoNull = { 0 }; + size_t headerSize; + size_t blockInfoSize; + size_t blockSize; + unsigned nbBlocks; + size_t frameSuffixSize; + size_t totalBound; + + if (frameInfoPtr==NULL) frameInfoPtr = &frameInfoNull; /* all parameters set to default */ + + headerSize = 7; /* basic header size (no option) including magic number */ + blockInfoSize = 4; /* basic blockInfo size (no option) for one block */ + + blockSize = LZ4F_getBlockSize(frameInfoPtr->maxBlockSizeID); + nbBlocks = (srcSize + (blockSize-1)) / blockSize; + blockInfoSize *= nbBlocks; /* total block info size */ + + frameSuffixSize = 4; /* basic frameSuffixSize (no option) */ + if (frameInfoPtr->contentChecksumFlag == contentChecksumEnabled) frameSuffixSize += 4; + + totalBound = headerSize + srcSize + blockInfoSize + frameSuffixSize; + if (totalBound < srcSize) return ERROR_srcSize_tooLarge; /* overflow error */ + + return totalBound; +} + + +/* LZ4F_compressFrame() + * Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.4.1, in a single step. + * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. + * You can get the minimum value of dstMaxSize by using LZ4F_compressFrameBound() + * If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode) + * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. + * The result of the function is the number of bytes written into dstBuffer. + * The function outputs an error code if it fails (can be tested using LZ4F_isError()) + */ +size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr) +{ + const LZ4F_frameInfo_t frameInfoNull = { 0 }; + const LZ4F_frameInfo_t* const frameInfoPtr = (preferencesPtr==NULL) ? &frameInfoNull : &(preferencesPtr->frameInfo); + LZ4F_compressionContext_t cctx; + LZ4F_errorCode_t errorCode; + BYTE* const dstStart = (BYTE*) dstBuffer; + BYTE* dstPtr = dstStart; + size_t blockSize = LZ4F_getBlockSize(frameInfoPtr->maxBlockSizeID); + unsigned nbBlocks = (srcSize + (blockSize-1)) / blockSize; + unsigned blockNb; + const BYTE* srcPtr = (const BYTE*) srcBuffer; + const size_t dstBlockSize = LZ4F_compressBound(blockSize, frameInfoPtr); + + + if (dstMaxSize < LZ4F_compressFrameBound(srcSize, frameInfoPtr)) + return ERROR_maxDstSize_tooSmall; + + errorCode = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION, preferencesPtr); + if (LZ4F_isError(errorCode)) return errorCode; + + errorCode = LZ4F_compressBegin(cctx, dstBuffer, dstMaxSize); /* write header */ + if (LZ4F_isError(errorCode)) return errorCode; + dstPtr += errorCode; /* header size */ + + for (blockNb=1; blockNb<nbBlocks; blockNb++) + { + errorCode = LZ4F_compress(cctx, dstPtr, dstBlockSize, srcPtr, blockSize, NULL); + if (LZ4F_isError(errorCode)) return errorCode; + srcPtr += blockSize; + dstPtr += errorCode; + } + + /* last block */ + blockSize = srcSize % blockSize; + errorCode = LZ4F_compress(cctx, dstPtr, dstBlockSize, srcPtr, blockSize, NULL); + if (LZ4F_isError(errorCode)) return errorCode; + dstPtr += errorCode; + + errorCode = LZ4F_compressEnd(cctx, dstPtr, dstMaxSize, NULL); /* flush last block, and generate suffix */ + if (LZ4F_isError(errorCode)) return errorCode; + dstPtr += errorCode; + + errorCode = LZ4F_freeCompressionContext(cctx); + if (LZ4F_isError(errorCode)) return errorCode; + + return (dstPtr - dstStart); +} @@ -46,10 +46,16 @@ extern "C" { /************************************** + Includes +**************************************/ +#include <stddef.h> /* size_t */ + + +/************************************** Error management **************************************/ typedef size_t LZ4F_errorCode_t; -typedef enum { OK_NoError = 0, OK_FrameEnd = 1 } LZ4F_successCodes; +typedef enum { OK_FrameEnd = 1 } LZ4F_successCodes; typedef enum { OK_NoError = 0, ERROR_GENERIC = 1, ERROR_maxBlockSize_invalid, ERROR_blockMode_invalid, ERROR_contentChecksumFlag_invalid, ERROR_compressionLevel_invalid, @@ -66,17 +72,17 @@ int LZ4F_isError(LZ4F_errorCode_t code); /* Basically : code > -ERROR_maxCode **************************************/ typedef enum { LZ4F_default=0, max64KB=4, max256KB=5, max1MB=6, max4MB=7} maxBlockSize_t; -typedef enum { LZ4F_default=0, blockLinked, blockIndependent} blockMode_t; -typedef enum { LZ4F_default=0, contentChecksumEnabled, noContentChecksum} contentChecksum_t; +typedef enum { blockLinked=1, blockIndependent} blockMode_t; +typedef enum { contentChecksumEnabled, noContentChecksum} contentChecksum_t; typedef struct { - maxBlockSize_t maxBlockSize; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ + maxBlockSize_t maxBlockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */ - contentChecksum_t contentChecksumFlag; /* contentChecksumEnabled, noContentChecksum ; 0 == default */ + contentChecksum_t contentChecksumFlag; /* contentChecksumEnabled (default), noContentChecksum ; */ } LZ4F_frameInfo_t; typedef struct { - LZ4F_frameInfo_t frameInfo + LZ4F_frameInfo_t frameInfo; unsigned compressionLevel; /* from 0 to 16 */ unsigned autoFlush; /* 1 == automatic flush after each call to LZ4F_compress() */ } LZ4F_preferences_t; @@ -86,9 +92,9 @@ typedef struct { /********************************** * Simple compression function * *********************************/ -size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_frameInfo_t* frameInfo); +size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_frameInfo_t* frameInfoPtr); -size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferences); +size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr); /* LZ4F_compressFrame() * Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.4.1, in a single step. * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. @@ -114,7 +120,7 @@ typedef struct { /* Resource Management */ #define LZ4F_VERSION 100 -LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContext, int version, const LZ4F_preferences_t* preferences); +LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, int version, const LZ4F_preferences_t* preferencesPtr); LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_compressionContext); /* LZ4F_createCompressionContext() : * The first thing to do is to create a compressionContext object, which will be used in all compression operations. @@ -129,7 +135,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp /* Compression */ -size_t LZ4F_compressBegin(LZ4F_compressionContext_t* compressionContextPtr, void* dstBuffer, size_t dstMaxSize); +size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize); /* LZ4F_compressBegin() : * will write the frame header into dstBuffer. * dstBuffer must be large enough to accomodate a header (dstMaxSize). Maximum header size is 15 bytes. @@ -137,15 +143,15 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t* compressionContextPtr, void * or an error code (can be tested using LZ4F_isError()) */ -size_t LZ4F_compressBound(size_t srcSize, const LZ4F_frameInfo_t* frameInfo); -size_t LZ4F_getMaxSrcSize(size_t dstMaxSize, const LZ4F_frameInfo_t* frameInfo); +size_t LZ4F_compressBound(size_t srcSize, const LZ4F_frameInfo_t* frameInfoPtr); +size_t LZ4F_getMaxSrcSize(size_t dstMaxSize, const LZ4F_frameInfo_t* frameInfoPtr); /* LZ4F_compressBound() : gives the size of Dst buffer given a srcSize to handle worst case situations. * LZ4F_getMaxSrcSize() : gives max allowed srcSize given dstMaxSize to handle worst case situations. * You can use dstMaxSize==0 to know the "natural" srcSize instead (block size). * The LZ4F_preferences_t structure is optional : you can provide NULL as argument, all preferences will then be set to default. */ -size_t LZ4F_compress(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptions); +size_t LZ4F_compress(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr); /* LZ4F_compress() * You can then call LZ4F_compress() repetitively to compress as much data as necessary. * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. @@ -157,7 +163,7 @@ size_t LZ4F_compress(LZ4F_compressionContext_t compressionContext, void* dstBuff * The function outputs an error code if it fails (can be tested using LZ4F_isError()) */ -size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptions); +size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr); /* LZ4F_flush() * Should you need to create compressed data immediately, without waiting for a block to be filled, * you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext. @@ -167,7 +173,7 @@ size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. */ -size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptions); +size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr); /* LZ4F_compressEnd() * When you want to properly finish the compressed frame, just call LZ4F_compressEnd(). * It will flush whatever data remained within compressionContext (like LZ4_flush()) @@ -191,7 +197,7 @@ typedef struct { /* Resource management */ -LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_compressionContext_t* LZ4F_decompressionContext); +LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_compressionContext_t* LZ4F_decompressionContextPtr); LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_compressionContext_t LZ4F_decompressionContext); /* LZ4F_createDecompressionContext() : * The first thing to do is to create a decompressionContext object, which will be used in all decompression operations. @@ -203,7 +209,7 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_compressionContext_t LZ4F_de /* Decompression */ -LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_frameInfo_t frameInfo, LZ4F_decompressionContext_t* decompressionContextPtr, void* srcBuffer, size_t* srcSize); +LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t decompressionContext, LZ4F_frameInfo_t* frameInfoPtr, const void* srcBuffer, size_t* srcSize); /* LZ4F_getFrameInfo() * This function decodes frame header information, such as blockSize. * It is optional : you could start by calling directly LZ4F_decompress() instead. @@ -213,7 +219,7 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_frameInfo_t frameInfo, LZ4F_decompressio * The function result is an error code which can be tested using LZ4F_isError(). */ -LZ4F_errorCode_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, void* dstBuffer, size_t* dstSize, const void* srcBuffer, size_t* srcSize, const LZ4F_decompressOptions_t* decompressOptions); +LZ4F_errorCode_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, void* dstBuffer, size_t* dstSize, const void* srcBuffer, size_t* srcSize, const LZ4F_decompressOptions_t* decompressOptionsPtr); /* LZ4F_decompress() * Call this function repetitively to regenerate data compressed within srcBuffer. * The function will attempt to decode *srcSize from srcBuffer, into dstBuffer of maximum size *dstSize. |