From db836b5519ab1498504dc27c989b71b87d3d4c4d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 13 Jul 2022 15:41:11 +0200 Subject: declare experimental prototype for LZ4F custom Memory manager --- lib/lz4frame.h | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 2c5a559..79522b6 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -545,6 +545,29 @@ typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult); + +/*! Custom memory allocation : + * These prototypes make it possible to pass your own allocation/free functions. + * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below. + * All allocation/free operations will be completed using these custom variants instead of regular ones. + */ +typedef void* (*LZ4F_allocFunction) (void* opaqueState, size_t size); +typedef void* (*LZ4F_callocFunction) (void* opaqueState, size_t size); +typedef void (*LZ4F_freeFunction) (void* opaqueState, void* address); +typedef struct { + LZ4F_allocFunction customAlloc; + LZ4F_callocFunction customCalloc; + LZ4F_freeFunction customFree; + void* opaqueState; +} LZ4F_customMem; +static +#ifdef __GNUC__ +__attribute__((__unused__)) +#endif +LZ4F_customMem const LZ4F_defaultCMem = { NULL, NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ + +LZ4FLIB_STATIC_API LZ4F_cctx* LZ4F_createCompressionContext_advanced(LZ4F_customMem customMem); + LZ4FLIB_STATIC_API size_t LZ4F_getBlockSize(unsigned); /*! LZ4F_uncompressedUpdate() : -- cgit v0.12 From 270529e80e12f0aa1778d64047472d049129c15d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 13 Jul 2022 18:37:53 +0200 Subject: implemented first custom memory manager interface for compression context only for the time being, using LZ4F_createCompressionContext_advanced(). Added basic test in frametest.c --- lib/lz4frame.c | 172 +++++++++++++++++++++++++++++++++++------------------- lib/lz4frame.h | 48 ++++++++------- tests/frametest.c | 60 +++++++++++++++++-- 3 files changed, 195 insertions(+), 85 deletions(-) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index aec2728..8db6b46 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -45,7 +45,7 @@ * Compiler Options **************************************/ #ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ #endif @@ -63,6 +63,19 @@ /*-************************************ +* Library declarations +**************************************/ +#define LZ4F_STATIC_LINKING_ONLY +#include "lz4frame.h" +#define LZ4_STATIC_LINKING_ONLY +#include "lz4.h" +#define LZ4_HC_STATIC_LINKING_ONLY +#include "lz4hc.h" +#define XXH_STATIC_LINKING_ONLY +#include "xxhash.h" + + +/*-************************************ * Memory routines **************************************/ /* @@ -70,7 +83,13 @@ * malloc(), calloc() and free() * towards another library or solution of their choice * by modifying below section. - */ +**/ + +#include /* memset, memcpy, memmove */ +#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */ +# define MEM_INIT(p,v,s) memset((p),(v),(s)) +#endif + #ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */ # include /* malloc, calloc, free */ # define ALLOC(s) malloc(s) @@ -78,23 +97,42 @@ # define FREEMEM(p) free(p) #endif -#include /* memset, memcpy, memmove */ -#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */ -# define MEM_INIT(p,v,s) memset((p),(v),(s)) -#endif +static void* LZ4F_calloc(size_t s, LZ4F_CustomMem cmem) +{ + /* custom calloc defined : use it */ + if (cmem.customCalloc != NULL) { + return cmem.customCalloc(cmem.opaqueState, s); + } + /* nothing defined : use default 's calloc() */ + if (cmem.customAlloc == NULL) { + return ALLOC_AND_ZERO(s); + } + /* only custom alloc defined : use it, and combine it with memset() */ + { void* const p = cmem.customAlloc(cmem.opaqueState, s); + if (p != NULL) MEM_INIT(p, 0, s); + return p; +} } +static void* LZ4F_malloc(size_t s, LZ4F_CustomMem cmem) +{ + /* custom malloc defined : use it */ + if (cmem.customAlloc != NULL) { + return cmem.customAlloc(cmem.opaqueState, s); + } + /* nothing defined : use default 's malloc() */ + return ALLOC(s); +} -/*-************************************ -* Library declarations -**************************************/ -#define LZ4F_STATIC_LINKING_ONLY -#include "lz4frame.h" -#define LZ4_STATIC_LINKING_ONLY -#include "lz4.h" -#define LZ4_HC_STATIC_LINKING_ONLY -#include "lz4hc.h" -#define XXH_STATIC_LINKING_ONLY -#include "xxhash.h" +static void LZ4F_free(void* p, LZ4F_CustomMem cmem) +{ + /* custom malloc defined : use it */ + if (cmem.customFree != NULL) { + cmem.customFree(cmem.opaqueState, p); + return; + } + /* nothing defined : use default 's free() */ + FREEMEM(p); +} /*-************************************ @@ -225,6 +263,7 @@ typedef enum { LZ4B_COMPRESSED, LZ4B_UNCOMPRESSED} LZ4F_blockCompression_t; typedef struct LZ4F_cctx_s { + LZ4F_CustomMem cmem; LZ4F_preferences_t prefs; U32 version; U32 cStage; @@ -441,27 +480,26 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, { size_t result; #if (LZ4F_HEAPMODE) - LZ4F_cctx_t *cctxPtr; + LZ4F_cctx_t* cctxPtr; result = LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION); FORWARD_IF_ERROR(result); #else LZ4F_cctx_t cctx; LZ4_stream_t lz4ctx; - LZ4F_cctx_t *cctxPtr = &cctx; + LZ4F_cctx_t* const cctxPtr = &cctx; - DEBUGLOG(4, "LZ4F_compressFrame"); MEM_INIT(&cctx, 0, sizeof(cctx)); cctx.version = LZ4F_VERSION; cctx.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */ - if (preferencesPtr == NULL || - preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN) - { + if ( preferencesPtr == NULL + || preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN ) { LZ4_initStream(&lz4ctx, sizeof(lz4ctx)); cctxPtr->lz4CtxPtr = &lz4ctx; cctxPtr->lz4CtxAlloc = 1; cctxPtr->lz4CtxState = 1; } #endif + DEBUGLOG(4, "LZ4F_compressFrame"); result = LZ4F_compressFrame_usingCDict(cctxPtr, dstBuffer, dstCapacity, srcBuffer, srcSize, @@ -470,10 +508,9 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, #if (LZ4F_HEAPMODE) LZ4F_freeCompressionContext(cctxPtr); #else - if (preferencesPtr != NULL && - preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) - { - FREEMEM(cctxPtr->lz4CtxPtr); + if ( preferencesPtr != NULL + && preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN ) { + LZ4F_free(cctxPtr->lz4CtxPtr, cctxPtr->cmem); } #endif return result; @@ -499,14 +536,14 @@ struct LZ4F_CDict_s { LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize) { const char* dictStart = (const char*)dictBuffer; - LZ4F_CDict* const cdict = (LZ4F_CDict*) ALLOC(sizeof(*cdict)); + LZ4F_CDict* const cdict = (LZ4F_CDict*)LZ4F_malloc(sizeof(*cdict), LZ4F_defaultCMem); DEBUGLOG(4, "LZ4F_createCDict"); if (!cdict) return NULL; if (dictSize > 64 KB) { dictStart += dictSize - 64 KB; dictSize = 64 KB; } - cdict->dictContent = ALLOC(dictSize); + cdict->dictContent = LZ4F_malloc(dictSize, LZ4F_defaultCMem); cdict->fastCtx = LZ4_createStream(); cdict->HCCtx = LZ4_createStreamHC(); if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) { @@ -523,10 +560,10 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize) void LZ4F_freeCDict(LZ4F_CDict* cdict) { if (cdict==NULL) return; /* support free on NULL */ - FREEMEM(cdict->dictContent); + LZ4F_free(cdict->dictContent, LZ4F_defaultCMem); LZ4_freeStream(cdict->fastCtx); LZ4_freeStreamHC(cdict->HCCtx); - FREEMEM(cdict); + LZ4F_free(cdict, LZ4F_defaultCMem); } @@ -534,6 +571,20 @@ void LZ4F_freeCDict(LZ4F_CDict* cdict) * Advanced compression functions ***********************************/ +LZ4F_cctx* +LZ4F_createCompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version) +{ + LZ4F_cctx* const cctxPtr = + (LZ4F_cctx*)LZ4F_calloc(sizeof(LZ4F_cctx), customMem); + if (cctxPtr==NULL) return NULL; + + cctxPtr->cmem = customMem; + cctxPtr->version = version; + cctxPtr->cStage = 0; /* Uninitialized. Next stage : init cctx */ + + return cctxPtr; +} + /*! LZ4F_createCompressionContext() : * The first thing to do is to create a compressionContext object, which will be used in all compression operations. * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. @@ -541,17 +592,16 @@ void LZ4F_freeCDict(LZ4F_CDict* cdict) * The function will provide a pointer to an allocated LZ4F_compressionContext_t object. * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. * Object can release its memory using LZ4F_freeCompressionContext(); - */ -LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** LZ4F_compressionContextPtr, unsigned version) +**/ +LZ4F_errorCode_t +LZ4F_createCompressionContext(LZ4F_cctx** LZ4F_compressionContextPtr, unsigned version) { - LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)ALLOC_AND_ZERO(sizeof(LZ4F_cctx_t)); - RETURN_ERROR_IF(cctxPtr==NULL, allocation_failed); - - cctxPtr->version = version; - cctxPtr->cStage = 0; /* Uninitialized. Next stage : init cctx */ - - *LZ4F_compressionContextPtr = cctxPtr; + assert(LZ4F_compressionContextPtr != NULL); /* considered a violation of narrow contract */ + /* in case it nonetheless happen in production */ + RETURN_ERROR_IF(LZ4F_compressionContextPtr == NULL, parameter_null); + *LZ4F_compressionContextPtr = LZ4F_createCompressionContext_advanced(LZ4F_defaultCMem, version); + RETURN_ERROR_IF(*LZ4F_compressionContextPtr==NULL, allocation_failed); return LZ4F_OK_NoError; } @@ -559,11 +609,10 @@ LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** LZ4F_compressionConte LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctxPtr) { if (cctxPtr != NULL) { /* support free on NULL */ - FREEMEM(cctxPtr->lz4CtxPtr); /* note: LZ4_streamHC_t and LZ4_stream_t are simple POD types */ - FREEMEM(cctxPtr->tmpBuff); - FREEMEM(cctxPtr); + LZ4F_free(cctxPtr->lz4CtxPtr, cctxPtr->cmem); /* note: LZ4_streamHC_t and LZ4_stream_t are simple POD types */ + LZ4F_free(cctxPtr->tmpBuff, cctxPtr->cmem); + LZ4F_free(cctxPtr, cctxPtr->cmem); } - return LZ4F_OK_NoError; } @@ -633,11 +682,17 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, int allocatedSize = ctxTypeID_to_size(cctxPtr->lz4CtxAlloc); if (allocatedSize < requiredSize) { /* not enough space allocated */ - FREEMEM(cctxPtr->lz4CtxPtr); + LZ4F_free(cctxPtr->lz4CtxPtr, cctxPtr->cmem); if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { - cctxPtr->lz4CtxPtr = LZ4_createStream(); + /* must take ownership of memory allocation, + * in order to respect custom allocator contract */ + cctxPtr->lz4CtxPtr = LZ4F_malloc(sizeof(LZ4_stream_t), cctxPtr->cmem); + if (cctxPtr->lz4CtxPtr) + LZ4_initStream(cctxPtr->lz4CtxPtr, sizeof(LZ4_stream_t)); } else { - cctxPtr->lz4CtxPtr = LZ4_createStreamHC(); + cctxPtr->lz4CtxPtr = LZ4F_malloc(sizeof(LZ4_streamHC_t), cctxPtr->cmem); + if (cctxPtr->lz4CtxPtr) + LZ4_initStreamHC(cctxPtr->lz4CtxPtr, sizeof(LZ4_streamHC_t)); } RETURN_ERROR_IF(cctxPtr->lz4CtxPtr == NULL, allocation_failed); cctxPtr->lz4CtxAlloc = ctxTypeID; @@ -652,8 +707,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); } cctxPtr->lz4CtxState = ctxTypeID; - } - } + } } /* Buffer Management */ if (cctxPtr->prefs.frameInfo.blockSizeID == 0) @@ -666,8 +720,8 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, if (cctxPtr->maxBufferSize < requiredBuffSize) { cctxPtr->maxBufferSize = 0; - FREEMEM(cctxPtr->tmpBuff); - cctxPtr->tmpBuff = (BYTE*)ALLOC_AND_ZERO(requiredBuffSize); + LZ4F_free(cctxPtr->tmpBuff, cctxPtr->cmem); + cctxPtr->tmpBuff = (BYTE*)LZ4F_calloc(requiredBuffSize, cctxPtr->cmem); RETURN_ERROR_IF(cctxPtr->tmpBuff == NULL, allocation_failed); cctxPtr->maxBufferSize = requiredBuffSize; } } @@ -1168,7 +1222,7 @@ struct LZ4F_dctx_s { */ LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber) { - LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOC_AND_ZERO(sizeof(LZ4F_dctx)); + LZ4F_dctx* const dctx = (LZ4F_dctx*)LZ4F_calloc(sizeof(LZ4F_dctx), LZ4F_defaultCMem); if (dctx == NULL) { /* failed allocation */ *LZ4F_decompressionContextPtr = NULL; RETURN_ERROR(allocation_failed); @@ -1184,9 +1238,9 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx) LZ4F_errorCode_t result = LZ4F_OK_NoError; if (dctx != NULL) { /* can accept NULL input, like free() */ result = (LZ4F_errorCode_t)dctx->dStage; - FREEMEM(dctx->tmpIn); - FREEMEM(dctx->tmpOutBuffer); - FREEMEM(dctx); + LZ4F_free(dctx->tmpIn, LZ4F_defaultCMem); + LZ4F_free(dctx->tmpOutBuffer, LZ4F_defaultCMem); + LZ4F_free(dctx, LZ4F_defaultCMem); } return result; } @@ -1550,11 +1604,11 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) ? 128 KB : 0); if (bufferNeeded > dctx->maxBufferSize) { /* tmp buffers too small */ dctx->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/ - FREEMEM(dctx->tmpIn); - dctx->tmpIn = (BYTE*)ALLOC(dctx->maxBlockSize + BFSize /* block checksum */); + LZ4F_free(dctx->tmpIn, LZ4F_defaultCMem); + dctx->tmpIn = (BYTE*)LZ4F_malloc(dctx->maxBlockSize + BFSize /* block checksum */, LZ4F_defaultCMem); RETURN_ERROR_IF(dctx->tmpIn == NULL, allocation_failed); - FREEMEM(dctx->tmpOutBuffer); - dctx->tmpOutBuffer= (BYTE*)ALLOC(bufferNeeded); + LZ4F_free(dctx->tmpOutBuffer, LZ4F_defaultCMem); + dctx->tmpOutBuffer= (BYTE*)LZ4F_malloc(bufferNeeded, LZ4F_defaultCMem); RETURN_ERROR_IF(dctx->tmpOutBuffer== NULL, allocation_failed); dctx->maxBufferSize = bufferNeeded; } } diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 79522b6..2c87564 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -243,17 +243,20 @@ typedef struct { LZ4FLIB_API unsigned LZ4F_getVersion(void); /*! LZ4F_createCompressionContext() : - * The first thing to do is to create a compressionContext object, - * which will keep track of operation state during streaming compression. - * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version. - * The version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL. - * The function will provide a pointer to a fully allocated LZ4F_cctx object. - * If @return != zero, there context creation failed. - * Once all streaming compression jobs are completed, - * the state object can be released using LZ4F_freeCompressionContext(). - * Note1 : LZ4F_freeCompressionContext() is always successful. Its return value can be ignored. - * Note2 : LZ4F_freeCompressionContext() works fine with NULL input pointers (do nothing). - */ + * The first thing to do is to create a compressionContext object, + * which will keep track of operation state during streaming compression. + * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version, + * and a pointer to LZ4F_cctx*, to write the resulting pointer into. + * @version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL. + * The function provides a pointer to a fully allocated LZ4F_cctx object. + * @cctxPtr MUST be != NULL. + * If @return != zero, context creation failed. + * A created compression context can be employed multiple times for consecutive streaming operations. + * Once all streaming compression jobs are completed, + * the state object can be released using LZ4F_freeCompressionContext(). + * Note1 : LZ4F_freeCompressionContext() is always successful. Its return value can be ignored. + * Note2 : LZ4F_freeCompressionContext() works fine with NULL input pointers (do nothing). +**/ LZ4FLIB_API LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version); LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); @@ -535,6 +538,7 @@ extern "C" { ITEM(ERROR_contentChecksum_invalid) \ ITEM(ERROR_frameDecoding_alreadyStarted) \ ITEM(ERROR_compressionState_uninitialized) \ + ITEM(ERROR_parameter_null) \ ITEM(ERROR_maxCode) #define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM, @@ -547,26 +551,26 @@ LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult); /*! Custom memory allocation : - * These prototypes make it possible to pass your own allocation/free functions. - * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below. + * These prototypes make it possible to pass custom allocation/free functions. + * LZ4F_customMem is provided at state creation time, using LZ4F_createCompressionContext_advanced() listed below. * All allocation/free operations will be completed using these custom variants instead of regular ones. */ -typedef void* (*LZ4F_allocFunction) (void* opaqueState, size_t size); -typedef void* (*LZ4F_callocFunction) (void* opaqueState, size_t size); -typedef void (*LZ4F_freeFunction) (void* opaqueState, void* address); +typedef void* (*LZ4F_AllocFunction) (void* opaqueState, size_t size); +typedef void* (*LZ4F_CallocFunction) (void* opaqueState, size_t size); +typedef void (*LZ4F_FreeFunction) (void* opaqueState, void* address); typedef struct { - LZ4F_allocFunction customAlloc; - LZ4F_callocFunction customCalloc; - LZ4F_freeFunction customFree; + LZ4F_AllocFunction customAlloc; + LZ4F_CallocFunction customCalloc; /* optional; when not defined, uses customAlloc + memset */ + LZ4F_FreeFunction customFree; void* opaqueState; -} LZ4F_customMem; +} LZ4F_CustomMem; static #ifdef __GNUC__ __attribute__((__unused__)) #endif -LZ4F_customMem const LZ4F_defaultCMem = { NULL, NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ +LZ4F_CustomMem const LZ4F_defaultCMem = { NULL, NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ -LZ4FLIB_STATIC_API LZ4F_cctx* LZ4F_createCompressionContext_advanced(LZ4F_customMem customMem); +LZ4FLIB_STATIC_API LZ4F_cctx* LZ4F_createCompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version); LZ4FLIB_STATIC_API size_t LZ4F_getBlockSize(unsigned); diff --git a/tests/frametest.c b/tests/frametest.c index a496c3c..b646f4b 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -104,12 +104,63 @@ static U32 use_pause = 0; #define MIN(a,b) ( (a) < (b) ? (a) : (b) ) #define MAX(a,b) ( (a) > (b) ? (a) : (b) ) +typedef struct { + int nbAllocs; +} Test_alloc_state; +static Test_alloc_state g_testAllocState = { 0 }; + +static void* dummy_malloc(void* state, size_t s) +{ + Test_alloc_state* const t = (Test_alloc_state*)state; + void* const p = malloc(s); + if (p==NULL) return NULL; + assert(t != NULL); + t->nbAllocs += 1; + DISPLAYLEVEL(6, "Allocating %zu bytes at address %p \n", s, p); + DISPLAYLEVEL(5, "nb allocated memory segments : %i \n", t->nbAllocs); + return p; +} + +static void* dummy_calloc(void* state, size_t s) +{ + Test_alloc_state* const t = (Test_alloc_state*)state; + void* const p = calloc(1, s); + if (p==NULL) return NULL; + assert(t != NULL); + t->nbAllocs += 1; + DISPLAYLEVEL(6, "Allocating and zeroing %zu bytes at address %p \n", s, p); + DISPLAYLEVEL(5, "nb allocated memory segments : %i \n", t->nbAllocs); + return p; +} + +static void dummy_free(void* state, void* p) +{ + Test_alloc_state* const t = (Test_alloc_state*)state; + if (p==NULL) { + DISPLAYLEVEL(5, "free() on NULL \n"); + return; + } + DISPLAYLEVEL(6, "freeing memory at address %p \n", p); + free(p); + assert(t != NULL); + DISPLAYLEVEL(5, "nb of allocated memory segments before this free : %i \n", t->nbAllocs); + assert(t->nbAllocs > 0); + t->nbAllocs -= 1; +} + +static const LZ4F_CustomMem lz4f_cmem_test = { + dummy_malloc, + dummy_calloc, + dummy_free, + &g_testAllocState +}; + + static clock_t FUZ_GetClockSpan(clock_t clockStart) { return clock() - clockStart; /* works even if overflow; max span ~ 30 mn */ } - #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) unsigned int FUZ_rand(unsigned int* src) { @@ -121,7 +172,6 @@ unsigned int FUZ_rand(unsigned int* src) return rand32 >> 5; } - #define FUZ_RAND15BITS (FUZ_rand(seed) & 0x7FFF) #define FUZ_RANDLENGTH ( (FUZ_rand(seed) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15) static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, double proba, U32* seed) @@ -515,7 +565,9 @@ int basicTests(U32 seed, double compressibility) /* dictID tests */ { size_t cErr; U32 const dictID = 0x99; - CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) ); + /* test advanced variant with custom allocator functions */ + cctx = LZ4F_createCompressionContext_advanced(lz4f_cmem_test, LZ4F_VERSION); + if (cctx==NULL) goto _output_error; DISPLAYLEVEL(3, "insert a dictID : "); memset(&prefs.frameInfo, 0, sizeof(prefs.frameInfo)); @@ -944,7 +996,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi clock_t const startClock = clock(); clock_t const clockDuration = duration_s * CLOCKS_PER_SEC; - /* Create buffers */ + /* Create states & buffers */ { size_t const creationStatus = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); } { size_t const creationStatus = LZ4F_createDecompressionContext(&dCtxNoise, LZ4F_VERSION); -- cgit v0.12 From a3c4f0d0a31580da09a955b6783203d6c383a9e4 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 13 Jul 2022 21:39:59 +0200 Subject: implemented LZ4F_createDecompressionContext_advanced() --- lib/lz4frame.c | 39 +++++++++++++++++++++++++-------------- lib/lz4frame.h | 8 +++++--- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 6b4d67d..c65256d 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -1193,6 +1193,7 @@ typedef enum { } dStage_t; struct LZ4F_dctx_s { + LZ4F_CustomMem cmem; LZ4F_frameInfo_t frameInfo; U32 version; dStage_t dStage; @@ -1214,22 +1215,32 @@ struct LZ4F_dctx_s { }; /* typedef'd to LZ4F_dctx in lz4frame.h */ +LZ4F_dctx* LZ4F_createDecompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version) +{ + LZ4F_dctx* const dctx = (LZ4F_dctx*)LZ4F_calloc(sizeof(LZ4F_dctx), customMem); + if (dctx == NULL) return NULL; + + dctx->cmem = customMem; + dctx->version = version; + return dctx; +} + /*! LZ4F_createDecompressionContext() : * Create a decompressionContext object, which will track all decompression operations. * Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object. * Object can later be released using LZ4F_freeDecompressionContext(). * @return : if != 0, there was an error during context creation. */ -LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber) +LZ4F_errorCode_t +LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber) { - LZ4F_dctx* const dctx = (LZ4F_dctx*)LZ4F_calloc(sizeof(LZ4F_dctx), LZ4F_defaultCMem); - if (dctx == NULL) { /* failed allocation */ - *LZ4F_decompressionContextPtr = NULL; + assert(LZ4F_decompressionContextPtr != NULL); /* violation of narrow contract */ + RETURN_ERROR_IF(LZ4F_decompressionContextPtr == NULL, parameter_null); /* in case it nonetheless happen in production */ + + *LZ4F_decompressionContextPtr = LZ4F_createDecompressionContext_advanced(LZ4F_defaultCMem, versionNumber); + if (*LZ4F_decompressionContextPtr == NULL) { /* failed allocation */ RETURN_ERROR(allocation_failed); } - - dctx->version = versionNumber; - *LZ4F_decompressionContextPtr = dctx; return LZ4F_OK_NoError; } @@ -1238,9 +1249,9 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx) LZ4F_errorCode_t result = LZ4F_OK_NoError; if (dctx != NULL) { /* can accept NULL input, like free() */ result = (LZ4F_errorCode_t)dctx->dStage; - LZ4F_free(dctx->tmpIn, LZ4F_defaultCMem); - LZ4F_free(dctx->tmpOutBuffer, LZ4F_defaultCMem); - LZ4F_free(dctx, LZ4F_defaultCMem); + LZ4F_free(dctx->tmpIn, dctx->cmem); + LZ4F_free(dctx->tmpOutBuffer, dctx->cmem); + LZ4F_free(dctx, dctx->cmem); } return result; } @@ -1604,11 +1615,11 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) ? 128 KB : 0); if (bufferNeeded > dctx->maxBufferSize) { /* tmp buffers too small */ dctx->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/ - LZ4F_free(dctx->tmpIn, LZ4F_defaultCMem); - dctx->tmpIn = (BYTE*)LZ4F_malloc(dctx->maxBlockSize + BFSize /* block checksum */, LZ4F_defaultCMem); + LZ4F_free(dctx->tmpIn, dctx->cmem); + dctx->tmpIn = (BYTE*)LZ4F_malloc(dctx->maxBlockSize + BFSize /* block checksum */, dctx->cmem); RETURN_ERROR_IF(dctx->tmpIn == NULL, allocation_failed); - LZ4F_free(dctx->tmpOutBuffer, LZ4F_defaultCMem); - dctx->tmpOutBuffer= (BYTE*)LZ4F_malloc(bufferNeeded, LZ4F_defaultCMem); + LZ4F_free(dctx->tmpOutBuffer, dctx->cmem); + dctx->tmpOutBuffer= (BYTE*)LZ4F_malloc(bufferNeeded, dctx->cmem); RETURN_ERROR_IF(dctx->tmpOutBuffer== NULL, allocation_failed); dctx->maxBufferSize = bufferNeeded; } } diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 1b4ee76..0d2ebf9 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -364,9 +364,10 @@ typedef struct { /*! LZ4F_createDecompressionContext() : * Create an LZ4F_dctx object, to track all decompression operations. - * The version provided MUST be LZ4F_VERSION. - * The function provides a pointer to an allocated and initialized LZ4F_dctx object. - * The result is an errorCode, which can be tested using LZ4F_isError(). + * @version provided MUST be LZ4F_VERSION. + * @dctxPtr MUST be valid. + * The function fills @dctxPtr with the value of a pointer to an allocated and initialized LZ4F_dctx object. + * The @return is an errorCode, which can be tested using LZ4F_isError(). * dctx memory can be released using LZ4F_freeDecompressionContext(); * Result of LZ4F_freeDecompressionContext() indicates current state of decompressionContext when being released. * That is, it should be == 0 if decompression has been completed fully and correctly. @@ -571,6 +572,7 @@ __attribute__((__unused__)) LZ4F_CustomMem const LZ4F_defaultCMem = { NULL, NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ LZ4FLIB_STATIC_API LZ4F_cctx* LZ4F_createCompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version); +LZ4FLIB_STATIC_API LZ4F_dctx* LZ4F_createDecompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version); /*! LZ4F_getBlockSize() : -- cgit v0.12 From e535d6424a2952dad6db73e2882abaef76a5e5e8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 14 Jul 2022 01:29:58 +0200 Subject: implemented LZ4F_createCDict_advanced() --- lib/lz4frame.c | 45 +++++++++++++++++++++++++++++---------------- lib/lz4frame.h | 50 ++++++++++++++++++++++++++------------------------ tests/frametest.c | 11 +++++++++-- 3 files changed, 64 insertions(+), 42 deletions(-) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index c65256d..258d85e 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -522,30 +522,31 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, *****************************************************/ struct LZ4F_CDict_s { + LZ4F_CustomMem cmem; void* dictContent; LZ4_stream_t* fastCtx; LZ4_streamHC_t* HCCtx; }; /* typedef'd to LZ4F_CDict within lz4frame_static.h */ -/*! LZ4F_createCDict() : - * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. - * LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. - * LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. - * @dictBuffer can be released after LZ4F_CDict creation, since its content is copied within CDict - * @return : digested dictionary for compression, or NULL if failed */ -LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize) +LZ4F_CDict* +LZ4F_createCDict_advanced(LZ4F_CustomMem cmem, const void* dictBuffer, size_t dictSize) { const char* dictStart = (const char*)dictBuffer; - LZ4F_CDict* const cdict = (LZ4F_CDict*)LZ4F_malloc(sizeof(*cdict), LZ4F_defaultCMem); - DEBUGLOG(4, "LZ4F_createCDict"); + LZ4F_CDict* const cdict = (LZ4F_CDict*)LZ4F_malloc(sizeof(*cdict), cmem); + DEBUGLOG(4, "LZ4F_createCDict_advanced"); if (!cdict) return NULL; + cdict->cmem = cmem; if (dictSize > 64 KB) { dictStart += dictSize - 64 KB; dictSize = 64 KB; } - cdict->dictContent = LZ4F_malloc(dictSize, LZ4F_defaultCMem); - cdict->fastCtx = LZ4_createStream(); - cdict->HCCtx = LZ4_createStreamHC(); + cdict->dictContent = LZ4F_malloc(dictSize, cmem); + cdict->fastCtx = (LZ4_stream_t*)LZ4F_malloc(sizeof(LZ4_stream_t), cmem); + if (cdict->fastCtx) + LZ4_initStream(cdict->fastCtx, sizeof(LZ4_stream_t)); + cdict->HCCtx = (LZ4_streamHC_t*)LZ4F_malloc(sizeof(LZ4_streamHC_t), cmem); + if (cdict->HCCtx) + LZ4_initStream(cdict->HCCtx, sizeof(LZ4_streamHC_t)); if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) { LZ4F_freeCDict(cdict); return NULL; @@ -557,13 +558,25 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize) return cdict; } +/*! LZ4F_createCDict() : + * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. + * LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. + * LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * @dictBuffer can be released after LZ4F_CDict creation, since its content is copied within CDict + * @return : digested dictionary for compression, or NULL if failed */ +LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize) +{ + DEBUGLOG(4, "LZ4F_createCDict"); + return LZ4F_createCDict_advanced(LZ4F_defaultCMem, dictBuffer, dictSize); +} + void LZ4F_freeCDict(LZ4F_CDict* cdict) { if (cdict==NULL) return; /* support free on NULL */ - LZ4F_free(cdict->dictContent, LZ4F_defaultCMem); - LZ4_freeStream(cdict->fastCtx); - LZ4_freeStreamHC(cdict->HCCtx); - LZ4F_free(cdict, LZ4F_defaultCMem); + LZ4F_free(cdict->dictContent, cdict->cmem); + LZ4F_free(cdict->fastCtx, cdict->cmem); + LZ4F_free(cdict->HCCtx, cdict->cmem); + LZ4F_free(cdict, cdict->cmem); } diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 0d2ebf9..74df963 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -551,30 +551,6 @@ typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult); -/*! Custom memory allocation : - * These prototypes make it possible to pass custom allocation/free functions. - * LZ4F_customMem is provided at state creation time, using LZ4F_createCompressionContext_advanced() listed below. - * All allocation/free operations will be completed using these custom variants instead of regular ones. - */ -typedef void* (*LZ4F_AllocFunction) (void* opaqueState, size_t size); -typedef void* (*LZ4F_CallocFunction) (void* opaqueState, size_t size); -typedef void (*LZ4F_FreeFunction) (void* opaqueState, void* address); -typedef struct { - LZ4F_AllocFunction customAlloc; - LZ4F_CallocFunction customCalloc; /* optional; when not defined, uses customAlloc + memset */ - LZ4F_FreeFunction customFree; - void* opaqueState; -} LZ4F_CustomMem; -static -#ifdef __GNUC__ -__attribute__((__unused__)) -#endif -LZ4F_CustomMem const LZ4F_defaultCMem = { NULL, NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ - -LZ4FLIB_STATIC_API LZ4F_cctx* LZ4F_createCompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version); -LZ4FLIB_STATIC_API LZ4F_dctx* LZ4F_createDecompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version); - - /*! LZ4F_getBlockSize() : * Return, in scalar format (size_t), * the maximum block size associated with blockSizeID. @@ -674,6 +650,32 @@ LZ4FLIB_STATIC_API size_t LZ4F_decompress_usingDict( const void* dict, size_t dictSize, const LZ4F_decompressOptions_t* decompressOptionsPtr); + +/*! Custom memory allocation : + * These prototypes make it possible to pass custom allocation/free functions. + * LZ4F_customMem is provided at state creation time, using LZ4F_create*_advanced() listed below. + * All allocation/free operations will be completed using these custom variants instead of regular ones. + */ +typedef void* (*LZ4F_AllocFunction) (void* opaqueState, size_t size); +typedef void* (*LZ4F_CallocFunction) (void* opaqueState, size_t size); +typedef void (*LZ4F_FreeFunction) (void* opaqueState, void* address); +typedef struct { + LZ4F_AllocFunction customAlloc; + LZ4F_CallocFunction customCalloc; /* optional; when not defined, uses customAlloc + memset */ + LZ4F_FreeFunction customFree; + void* opaqueState; +} LZ4F_CustomMem; +static +#ifdef __GNUC__ +__attribute__((__unused__)) +#endif +LZ4F_CustomMem const LZ4F_defaultCMem = { NULL, NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ + +LZ4FLIB_STATIC_API LZ4F_cctx* LZ4F_createCompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version); +LZ4FLIB_STATIC_API LZ4F_dctx* LZ4F_createDecompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version); +LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict_advanced(LZ4F_CustomMem customMem, const void* dictBuffer, size_t dictSize); + + #if defined (__cplusplus) } #endif diff --git a/tests/frametest.c b/tests/frametest.c index 00b8acb..ec2cb12 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -143,9 +143,9 @@ static void dummy_free(void* state, void* p) DISPLAYLEVEL(6, "freeing memory at address %p \n", p); free(p); assert(t != NULL); - DISPLAYLEVEL(5, "nb of allocated memory segments before this free : %i \n", t->nbAllocs); - assert(t->nbAllocs > 0); t->nbAllocs -= 1; + DISPLAYLEVEL(5, "nb of allocated memory segments after this free : %i \n", t->nbAllocs); + assert(t->nbAllocs >= 0); } static const LZ4F_CustomMem lz4f_cmem_test = { @@ -595,6 +595,13 @@ int basicTests(U32 seed, double compressibility) if (cdict == NULL) goto _output_error; CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) ); + DISPLAYLEVEL(3, "Testing LZ4F_createCDict_advanced : "); + { LZ4F_CDict* const cda = LZ4F_createCDict_advanced(lz4f_cmem_test, CNBuffer, dictSize); + if (cda == NULL) goto _output_error; + LZ4F_freeCDict(cda); + } + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with NULL dict : "); CHECK_V(cSizeNoDict, LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity, -- cgit v0.12