diff options
-rw-r--r-- | lib/lz4frame.c | 172 | ||||
-rw-r--r-- | lib/lz4frame.h | 50 | ||||
-rw-r--r-- | tests/frametest.c | 60 |
3 files changed, 208 insertions, 74 deletions
diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 5373083..6b4d67d 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -45,7 +45,7 @@ * Compiler Options **************************************/ #ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ #endif @@ -63,6 +63,19 @@ /*-************************************ +* Library declarations +**************************************/ +#define LZ4F_STATIC_LINKING_ONLY +#include "lz4frame.h" +#define LZ4_STATIC_LINKING_ONLY +#include "lz4.h" +#define LZ4_HC_STATIC_LINKING_ONLY +#include "lz4hc.h" +#define XXH_STATIC_LINKING_ONLY +#include "xxhash.h" + + +/*-************************************ * Memory routines **************************************/ /* @@ -70,7 +83,13 @@ * malloc(), calloc() and free() * towards another library or solution of their choice * by modifying below section. - */ +**/ + +#include <string.h> /* memset, memcpy, memmove */ +#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */ +# define MEM_INIT(p,v,s) memset((p),(v),(s)) +#endif + #ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */ # include <stdlib.h> /* malloc, calloc, free */ # define ALLOC(s) malloc(s) @@ -78,23 +97,42 @@ # define FREEMEM(p) free(p) #endif -#include <string.h> /* memset, memcpy, memmove */ -#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */ -# define MEM_INIT(p,v,s) memset((p),(v),(s)) -#endif +static void* LZ4F_calloc(size_t s, LZ4F_CustomMem cmem) +{ + /* custom calloc defined : use it */ + if (cmem.customCalloc != NULL) { + return cmem.customCalloc(cmem.opaqueState, s); + } + /* nothing defined : use default <stdlib.h>'s calloc() */ + if (cmem.customAlloc == NULL) { + return ALLOC_AND_ZERO(s); + } + /* only custom alloc defined : use it, and combine it with memset() */ + { void* const p = cmem.customAlloc(cmem.opaqueState, s); + if (p != NULL) MEM_INIT(p, 0, s); + return p; +} } +static void* LZ4F_malloc(size_t s, LZ4F_CustomMem cmem) +{ + /* custom malloc defined : use it */ + if (cmem.customAlloc != NULL) { + return cmem.customAlloc(cmem.opaqueState, s); + } + /* nothing defined : use default <stdlib.h>'s malloc() */ + return ALLOC(s); +} -/*-************************************ -* Library declarations -**************************************/ -#define LZ4F_STATIC_LINKING_ONLY -#include "lz4frame.h" -#define LZ4_STATIC_LINKING_ONLY -#include "lz4.h" -#define LZ4_HC_STATIC_LINKING_ONLY -#include "lz4hc.h" -#define XXH_STATIC_LINKING_ONLY -#include "xxhash.h" +static void LZ4F_free(void* p, LZ4F_CustomMem cmem) +{ + /* custom malloc defined : use it */ + if (cmem.customFree != NULL) { + cmem.customFree(cmem.opaqueState, p); + return; + } + /* nothing defined : use default <stdlib.h>'s free() */ + FREEMEM(p); +} /*-************************************ @@ -225,6 +263,7 @@ typedef enum { LZ4B_COMPRESSED, LZ4B_UNCOMPRESSED} LZ4F_blockCompression_t; typedef struct LZ4F_cctx_s { + LZ4F_CustomMem cmem; LZ4F_preferences_t prefs; U32 version; U32 cStage; @@ -441,27 +480,26 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, { size_t result; #if (LZ4F_HEAPMODE) - LZ4F_cctx_t *cctxPtr; + LZ4F_cctx_t* cctxPtr; result = LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION); FORWARD_IF_ERROR(result); #else LZ4F_cctx_t cctx; LZ4_stream_t lz4ctx; - LZ4F_cctx_t *cctxPtr = &cctx; + LZ4F_cctx_t* const cctxPtr = &cctx; - DEBUGLOG(4, "LZ4F_compressFrame"); MEM_INIT(&cctx, 0, sizeof(cctx)); cctx.version = LZ4F_VERSION; cctx.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */ - if (preferencesPtr == NULL || - preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN) - { + if ( preferencesPtr == NULL + || preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN ) { LZ4_initStream(&lz4ctx, sizeof(lz4ctx)); cctxPtr->lz4CtxPtr = &lz4ctx; cctxPtr->lz4CtxAlloc = 1; cctxPtr->lz4CtxState = 1; } #endif + DEBUGLOG(4, "LZ4F_compressFrame"); result = LZ4F_compressFrame_usingCDict(cctxPtr, dstBuffer, dstCapacity, srcBuffer, srcSize, @@ -470,10 +508,9 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, #if (LZ4F_HEAPMODE) LZ4F_freeCompressionContext(cctxPtr); #else - if (preferencesPtr != NULL && - preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) - { - FREEMEM(cctxPtr->lz4CtxPtr); + if ( preferencesPtr != NULL + && preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN ) { + LZ4F_free(cctxPtr->lz4CtxPtr, cctxPtr->cmem); } #endif return result; @@ -499,14 +536,14 @@ struct LZ4F_CDict_s { LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize) { const char* dictStart = (const char*)dictBuffer; - LZ4F_CDict* const cdict = (LZ4F_CDict*) ALLOC(sizeof(*cdict)); + LZ4F_CDict* const cdict = (LZ4F_CDict*)LZ4F_malloc(sizeof(*cdict), LZ4F_defaultCMem); DEBUGLOG(4, "LZ4F_createCDict"); if (!cdict) return NULL; if (dictSize > 64 KB) { dictStart += dictSize - 64 KB; dictSize = 64 KB; } - cdict->dictContent = ALLOC(dictSize); + cdict->dictContent = LZ4F_malloc(dictSize, LZ4F_defaultCMem); cdict->fastCtx = LZ4_createStream(); cdict->HCCtx = LZ4_createStreamHC(); if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) { @@ -523,10 +560,10 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize) void LZ4F_freeCDict(LZ4F_CDict* cdict) { if (cdict==NULL) return; /* support free on NULL */ - FREEMEM(cdict->dictContent); + LZ4F_free(cdict->dictContent, LZ4F_defaultCMem); LZ4_freeStream(cdict->fastCtx); LZ4_freeStreamHC(cdict->HCCtx); - FREEMEM(cdict); + LZ4F_free(cdict, LZ4F_defaultCMem); } @@ -534,6 +571,20 @@ void LZ4F_freeCDict(LZ4F_CDict* cdict) * Advanced compression functions ***********************************/ +LZ4F_cctx* +LZ4F_createCompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version) +{ + LZ4F_cctx* const cctxPtr = + (LZ4F_cctx*)LZ4F_calloc(sizeof(LZ4F_cctx), customMem); + if (cctxPtr==NULL) return NULL; + + cctxPtr->cmem = customMem; + cctxPtr->version = version; + cctxPtr->cStage = 0; /* Uninitialized. Next stage : init cctx */ + + return cctxPtr; +} + /*! LZ4F_createCompressionContext() : * The first thing to do is to create a compressionContext object, which will be used in all compression operations. * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. @@ -541,17 +592,16 @@ void LZ4F_freeCDict(LZ4F_CDict* cdict) * The function will provide a pointer to an allocated LZ4F_compressionContext_t object. * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. * Object can release its memory using LZ4F_freeCompressionContext(); - */ -LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** LZ4F_compressionContextPtr, unsigned version) +**/ +LZ4F_errorCode_t +LZ4F_createCompressionContext(LZ4F_cctx** LZ4F_compressionContextPtr, unsigned version) { - LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)ALLOC_AND_ZERO(sizeof(LZ4F_cctx_t)); - RETURN_ERROR_IF(cctxPtr==NULL, allocation_failed); - - cctxPtr->version = version; - cctxPtr->cStage = 0; /* Uninitialized. Next stage : init cctx */ - - *LZ4F_compressionContextPtr = cctxPtr; + assert(LZ4F_compressionContextPtr != NULL); /* considered a violation of narrow contract */ + /* in case it nonetheless happen in production */ + RETURN_ERROR_IF(LZ4F_compressionContextPtr == NULL, parameter_null); + *LZ4F_compressionContextPtr = LZ4F_createCompressionContext_advanced(LZ4F_defaultCMem, version); + RETURN_ERROR_IF(*LZ4F_compressionContextPtr==NULL, allocation_failed); return LZ4F_OK_NoError; } @@ -559,11 +609,10 @@ LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** LZ4F_compressionConte LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctxPtr) { if (cctxPtr != NULL) { /* support free on NULL */ - FREEMEM(cctxPtr->lz4CtxPtr); /* note: LZ4_streamHC_t and LZ4_stream_t are simple POD types */ - FREEMEM(cctxPtr->tmpBuff); - FREEMEM(cctxPtr); + LZ4F_free(cctxPtr->lz4CtxPtr, cctxPtr->cmem); /* note: LZ4_streamHC_t and LZ4_stream_t are simple POD types */ + LZ4F_free(cctxPtr->tmpBuff, cctxPtr->cmem); + LZ4F_free(cctxPtr, cctxPtr->cmem); } - return LZ4F_OK_NoError; } @@ -633,11 +682,17 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, int allocatedSize = ctxTypeID_to_size(cctxPtr->lz4CtxAlloc); if (allocatedSize < requiredSize) { /* not enough space allocated */ - FREEMEM(cctxPtr->lz4CtxPtr); + LZ4F_free(cctxPtr->lz4CtxPtr, cctxPtr->cmem); if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { - cctxPtr->lz4CtxPtr = LZ4_createStream(); + /* must take ownership of memory allocation, + * in order to respect custom allocator contract */ + cctxPtr->lz4CtxPtr = LZ4F_malloc(sizeof(LZ4_stream_t), cctxPtr->cmem); + if (cctxPtr->lz4CtxPtr) + LZ4_initStream(cctxPtr->lz4CtxPtr, sizeof(LZ4_stream_t)); } else { - cctxPtr->lz4CtxPtr = LZ4_createStreamHC(); + cctxPtr->lz4CtxPtr = LZ4F_malloc(sizeof(LZ4_streamHC_t), cctxPtr->cmem); + if (cctxPtr->lz4CtxPtr) + LZ4_initStreamHC(cctxPtr->lz4CtxPtr, sizeof(LZ4_streamHC_t)); } RETURN_ERROR_IF(cctxPtr->lz4CtxPtr == NULL, allocation_failed); cctxPtr->lz4CtxAlloc = ctxTypeID; @@ -652,8 +707,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); } cctxPtr->lz4CtxState = ctxTypeID; - } - } + } } /* Buffer Management */ if (cctxPtr->prefs.frameInfo.blockSizeID == 0) @@ -666,8 +720,8 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, if (cctxPtr->maxBufferSize < requiredBuffSize) { cctxPtr->maxBufferSize = 0; - FREEMEM(cctxPtr->tmpBuff); - cctxPtr->tmpBuff = (BYTE*)ALLOC_AND_ZERO(requiredBuffSize); + LZ4F_free(cctxPtr->tmpBuff, cctxPtr->cmem); + cctxPtr->tmpBuff = (BYTE*)LZ4F_calloc(requiredBuffSize, cctxPtr->cmem); RETURN_ERROR_IF(cctxPtr->tmpBuff == NULL, allocation_failed); cctxPtr->maxBufferSize = requiredBuffSize; } } @@ -1168,7 +1222,7 @@ struct LZ4F_dctx_s { */ LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber) { - LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOC_AND_ZERO(sizeof(LZ4F_dctx)); + LZ4F_dctx* const dctx = (LZ4F_dctx*)LZ4F_calloc(sizeof(LZ4F_dctx), LZ4F_defaultCMem); if (dctx == NULL) { /* failed allocation */ *LZ4F_decompressionContextPtr = NULL; RETURN_ERROR(allocation_failed); @@ -1184,9 +1238,9 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx) LZ4F_errorCode_t result = LZ4F_OK_NoError; if (dctx != NULL) { /* can accept NULL input, like free() */ result = (LZ4F_errorCode_t)dctx->dStage; - FREEMEM(dctx->tmpIn); - FREEMEM(dctx->tmpOutBuffer); - FREEMEM(dctx); + LZ4F_free(dctx->tmpIn, LZ4F_defaultCMem); + LZ4F_free(dctx->tmpOutBuffer, LZ4F_defaultCMem); + LZ4F_free(dctx, LZ4F_defaultCMem); } return result; } @@ -1550,11 +1604,11 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) ? 128 KB : 0); if (bufferNeeded > dctx->maxBufferSize) { /* tmp buffers too small */ dctx->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/ - FREEMEM(dctx->tmpIn); - dctx->tmpIn = (BYTE*)ALLOC(dctx->maxBlockSize + BFSize /* block checksum */); + LZ4F_free(dctx->tmpIn, LZ4F_defaultCMem); + dctx->tmpIn = (BYTE*)LZ4F_malloc(dctx->maxBlockSize + BFSize /* block checksum */, LZ4F_defaultCMem); RETURN_ERROR_IF(dctx->tmpIn == NULL, allocation_failed); - FREEMEM(dctx->tmpOutBuffer); - dctx->tmpOutBuffer= (BYTE*)ALLOC(bufferNeeded); + LZ4F_free(dctx->tmpOutBuffer, LZ4F_defaultCMem); + dctx->tmpOutBuffer= (BYTE*)LZ4F_malloc(bufferNeeded, LZ4F_defaultCMem); RETURN_ERROR_IF(dctx->tmpOutBuffer== NULL, allocation_failed); dctx->maxBufferSize = bufferNeeded; } } diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 1de09d8..1b4ee76 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -243,17 +243,20 @@ typedef struct { LZ4FLIB_API unsigned LZ4F_getVersion(void); /*! LZ4F_createCompressionContext() : - * The first thing to do is to create a compressionContext object, - * which will keep track of operation state during streaming compression. - * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version. - * The version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL. - * The function will provide a pointer to a fully allocated LZ4F_cctx object. - * If @return != zero, there context creation failed. - * Once all streaming compression jobs are completed, - * the state object can be released using LZ4F_freeCompressionContext(). - * Note1 : LZ4F_freeCompressionContext() is always successful. Its return value can be ignored. - * Note2 : LZ4F_freeCompressionContext() works fine with NULL input pointers (do nothing). - */ + * The first thing to do is to create a compressionContext object, + * which will keep track of operation state during streaming compression. + * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version, + * and a pointer to LZ4F_cctx*, to write the resulting pointer into. + * @version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL. + * The function provides a pointer to a fully allocated LZ4F_cctx object. + * @cctxPtr MUST be != NULL. + * If @return != zero, context creation failed. + * A created compression context can be employed multiple times for consecutive streaming operations. + * Once all streaming compression jobs are completed, + * the state object can be released using LZ4F_freeCompressionContext(). + * Note1 : LZ4F_freeCompressionContext() is always successful. Its return value can be ignored. + * Note2 : LZ4F_freeCompressionContext() works fine with NULL input pointers (do nothing). +**/ LZ4FLIB_API LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version); LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); @@ -535,6 +538,7 @@ extern "C" { ITEM(ERROR_contentChecksum_invalid) \ ITEM(ERROR_frameDecoding_alreadyStarted) \ ITEM(ERROR_compressionState_uninitialized) \ + ITEM(ERROR_parameter_null) \ ITEM(ERROR_maxCode) #define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM, @@ -545,6 +549,30 @@ typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult); + +/*! Custom memory allocation : + * These prototypes make it possible to pass custom allocation/free functions. + * LZ4F_customMem is provided at state creation time, using LZ4F_createCompressionContext_advanced() listed below. + * All allocation/free operations will be completed using these custom variants instead of regular <stdlib.h> ones. + */ +typedef void* (*LZ4F_AllocFunction) (void* opaqueState, size_t size); +typedef void* (*LZ4F_CallocFunction) (void* opaqueState, size_t size); +typedef void (*LZ4F_FreeFunction) (void* opaqueState, void* address); +typedef struct { + LZ4F_AllocFunction customAlloc; + LZ4F_CallocFunction customCalloc; /* optional; when not defined, uses customAlloc + memset */ + LZ4F_FreeFunction customFree; + void* opaqueState; +} LZ4F_CustomMem; +static +#ifdef __GNUC__ +__attribute__((__unused__)) +#endif +LZ4F_CustomMem const LZ4F_defaultCMem = { NULL, NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ + +LZ4FLIB_STATIC_API LZ4F_cctx* LZ4F_createCompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version); + + /*! LZ4F_getBlockSize() : * Return, in scalar format (size_t), * the maximum block size associated with blockSizeID. diff --git a/tests/frametest.c b/tests/frametest.c index 58eac38..00b8acb 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -104,12 +104,63 @@ static U32 use_pause = 0; #define MIN(a,b) ( (a) < (b) ? (a) : (b) ) #define MAX(a,b) ( (a) > (b) ? (a) : (b) ) +typedef struct { + int nbAllocs; +} Test_alloc_state; +static Test_alloc_state g_testAllocState = { 0 }; + +static void* dummy_malloc(void* state, size_t s) +{ + Test_alloc_state* const t = (Test_alloc_state*)state; + void* const p = malloc(s); + if (p==NULL) return NULL; + assert(t != NULL); + t->nbAllocs += 1; + DISPLAYLEVEL(6, "Allocating %zu bytes at address %p \n", s, p); + DISPLAYLEVEL(5, "nb allocated memory segments : %i \n", t->nbAllocs); + return p; +} + +static void* dummy_calloc(void* state, size_t s) +{ + Test_alloc_state* const t = (Test_alloc_state*)state; + void* const p = calloc(1, s); + if (p==NULL) return NULL; + assert(t != NULL); + t->nbAllocs += 1; + DISPLAYLEVEL(6, "Allocating and zeroing %zu bytes at address %p \n", s, p); + DISPLAYLEVEL(5, "nb allocated memory segments : %i \n", t->nbAllocs); + return p; +} + +static void dummy_free(void* state, void* p) +{ + Test_alloc_state* const t = (Test_alloc_state*)state; + if (p==NULL) { + DISPLAYLEVEL(5, "free() on NULL \n"); + return; + } + DISPLAYLEVEL(6, "freeing memory at address %p \n", p); + free(p); + assert(t != NULL); + DISPLAYLEVEL(5, "nb of allocated memory segments before this free : %i \n", t->nbAllocs); + assert(t->nbAllocs > 0); + t->nbAllocs -= 1; +} + +static const LZ4F_CustomMem lz4f_cmem_test = { + dummy_malloc, + dummy_calloc, + dummy_free, + &g_testAllocState +}; + + static clock_t FUZ_GetClockSpan(clock_t clockStart) { return clock() - clockStart; /* works even if overflow; max span ~ 30 mn */ } - #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) unsigned int FUZ_rand(unsigned int* src) { @@ -121,7 +172,6 @@ unsigned int FUZ_rand(unsigned int* src) return rand32 >> 5; } - #define FUZ_RAND15BITS (FUZ_rand(seed) & 0x7FFF) #define FUZ_RANDLENGTH ( (FUZ_rand(seed) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15) static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, double proba, U32* seed) @@ -515,7 +565,9 @@ int basicTests(U32 seed, double compressibility) /* dictID tests */ { size_t cErr; U32 const dictID = 0x99; - CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) ); + /* test advanced variant with custom allocator functions */ + cctx = LZ4F_createCompressionContext_advanced(lz4f_cmem_test, LZ4F_VERSION); + if (cctx==NULL) goto _output_error; DISPLAYLEVEL(3, "insert a dictID : "); memset(&prefs.frameInfo, 0, sizeof(prefs.frameInfo)); @@ -944,7 +996,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi clock_t const startClock = clock(); clock_t const clockDuration = duration_s * CLOCKS_PER_SEC; - /* Create buffers */ + /* Create states & buffers */ { size_t const creationStatus = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); } { size_t const creationStatus = LZ4F_createDecompressionContext(&dCtxNoise, LZ4F_VERSION); |