From 430b7d32b3f804ece3be95f0b36fb584947d6c73 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 16 Mar 2017 02:16:24 -0700 Subject: created LZ4_HC_STATIC_LINKING_ONLY section where are exposed new prototypes *_destSize() --- lib/lz4hc.c | 99 +++++++++++++++++++++++++--------------------------------- lib/lz4hc.h | 57 +++++++++++++++++++++++++++------ tests/fuzzer.c | 7 +++-- 3 files changed, 94 insertions(+), 69 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 290e575..b1e77ea 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -38,51 +38,37 @@ * Tuning Parameter ***************************************/ -/*! - * HEAPMODE : - * Select how default compression function will allocate workplace memory, - * in stack (0:fastest), or in heap (1:requires malloc()). - * Since workplace is rather large, heap mode is recommended. +/*! HEAPMODE : + * Select how default compression function will allocate workplace memory, + * in stack (0:fastest), or in heap (1:requires malloc()). + * Since workplace is rather large, heap mode is recommended. */ #ifndef LZ4HC_HEAPMODE # define LZ4HC_HEAPMODE 1 #endif -/* ************************************* -* Dependency -***************************************/ +/*=== Dependency ===*/ #include "lz4hc.h" -/* ************************************* -* Local Compiler Options -***************************************/ +/*=== Common LZ4 definitions ===*/ #if defined(__GNUC__) # pragma GCC diagnostic ignored "-Wunused-function" #endif - #if defined (__clang__) # pragma clang diagnostic ignored "-Wunused-function" #endif - -/* ************************************* -* Common LZ4 definition -***************************************/ #define LZ4_COMMONDEFS_ONLY #include "lz4.c" -/* ************************************* -* Local Constants -***************************************/ +/*=== Constants ===*/ #define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH) -/************************************** -* Local Macros -**************************************/ +/*=== Macros ===*/ #define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG)) #define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */ #define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */ @@ -141,8 +127,8 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* hc4, /* In const U32 dictLimit = hc4->dictLimit; const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); U32 matchIndex; - int nbAttempts=maxNbAttempts; - size_t ml=0; + int nbAttempts = maxNbAttempts; + size_t ml = 0; /* HC4 match finder */ LZ4HC_Insert(hc4, ip); @@ -223,9 +209,7 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( longest = (int)mlt; *matchpos = matchPtr+back; *startpos = ip+back; - } - } - } + } } } } else { const BYTE* const matchPtr = dictBase + matchIndex; if (LZ4_read32(matchPtr) == LZ4_read32(ip)) { @@ -279,10 +263,9 @@ FORCE_INLINE int LZ4HC_encodeSequence ( length = (size_t)(*ip - *anchor); token = (*op)++; if ((limit) && ((*op + (length >> 8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */ - if (length >= RUN_MASK) { - size_t len; + if (length >= RUN_MASK) { + size_t len = length - RUN_MASK; *token = (RUN_MASK << ML_BITS); - len = length - RUN_MASK; for(; len >= 255 ; len -= 255) *(*op)++ = 255; *(*op)++ = (BYTE)len; } else { @@ -316,8 +299,10 @@ FORCE_INLINE int LZ4HC_encodeSequence ( return 0; } +/* btopt */ #include "lz4opt.h" + static int LZ4HC_compress_hashChain ( LZ4HC_CCtx_internal* const ctx, const char* const source, @@ -399,11 +384,9 @@ _Search2: } _Search3: - /* - * Currently we have : - * ml2 > ml1, and - * ip1+3 <= ip2 (usually < ip1+ml1) - */ + /* At this stage, we have : + * ml2 > ml1, and + * ip1+3 <= ip2 (usually < ip1+ml1) */ if ((start2 - ip) < OPTIMAL_ML) { int correction; int new_ml = ml; @@ -502,10 +485,9 @@ _Search3: _last_literals: /* Encode Last Literals */ - { size_t lastRunSize, litLength, totalSize; - lastRunSize = (size_t)(iend - anchor); /* literals */ - litLength = (lastRunSize + 255 - RUN_MASK) / 255; - totalSize = 1 + litLength + lastRunSize; + { size_t lastRunSize = (size_t)(iend - anchor); /* literals */ + size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255; + size_t const totalSize = 1 + litLength + lastRunSize; if (limit == limitedDestSize) oend += LASTLITERALS; /* restore correct value */ if (limit && (op + totalSize > oend)) { if (limit == limitedOutput) return 0; /* Check output limit */ @@ -544,8 +526,8 @@ static int LZ4HC_getSearchNum(int compressionLevel) { switch (compressionLevel) { default: return 0; /* unused */ - case 11: return 128; - case 12: return 1<<10; + case 11: return 128; + case 12: return 1<<10; } } @@ -563,10 +545,15 @@ static int LZ4HC_compress_generic ( if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT; if (compressionLevel > 9) { switch (compressionLevel) { - case 10: return LZ4HC_compress_hashChain(ctx, source, dest, &srcSize, maxOutputSize, 1 << (16-1), limit); - case 11: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 128, 0); + case 10: + return LZ4HC_compress_hashChain(ctx, source, dest, &srcSize, maxOutputSize, 1 << (16-1), limit); + case 11: + ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); + return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 128, 0); default: - case 12: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM, 1); + case 12: + ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); + return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM, 1); } } return LZ4HC_compress_hashChain(ctx, source, dest, &srcSize, maxOutputSize, 1 << (compressionLevel-1), limit); @@ -577,7 +564,7 @@ int LZ4_sizeofStateHC(void) { return sizeof(LZ4_streamHC_t); } int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel) { - LZ4HC_CCtx_internal* ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; + LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ LZ4HC_init (ctx, (const BYTE*)src); if (maxDstSize < LZ4_compressBound(srcSize)) @@ -622,7 +609,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) { - LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse; + LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; if (dictSize > 64 KB) { dictionary += dictSize - 64 KB; dictSize = 64 KB; @@ -659,7 +646,7 @@ static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize, limitedOutput_directive limit) { - LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse; + LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; /* auto-init if forgotten */ if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) source); @@ -697,7 +684,7 @@ int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* sourc int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int* sourceSizePtr, int targetDestSize) { - LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse; + LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; unsigned maxNbAttempts = 1 << (ctxPtr->compressionLevel - 1); /* destSize: always auto-init */ @@ -706,6 +693,14 @@ int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const ch return LZ4HC_compress_hashChain(ctxPtr, source, dest, sourceSizePtr, targetDestSize, maxNbAttempts, limitedDestSize); } +int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int compressionLevel) +{ + LZ4HC_CCtx_internal * const ctx = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse; + unsigned maxNbAttempts = 1 << (compressionLevel - 1); + LZ4HC_init(ctx, (const BYTE*) source); + return LZ4HC_compress_hashChain(ctx, source, dest, sourceSizePtr, targetDestSize, maxNbAttempts, limitedDestSize); +} + /* dictionary saving */ @@ -784,11 +779,3 @@ char* LZ4_slideInputBufferHC(void* LZ4HC_Data) int const dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB); return (char*)(hc4->inputBuffer + dictSize); } - -int LZ4_compressHC_destSize(void* LZ4HC_Data, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int compressionLevel) -{ - LZ4HC_CCtx_internal * const ctx = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse; - unsigned maxNbAttempts = 1 << (compressionLevel - 1); - LZ4HC_init(ctx, (const BYTE*) source); - return LZ4HC_compress_hashChain(ctx, source, dest, sourceSizePtr, targetDestSize, maxNbAttempts, limitedDestSize); -} diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 13112f9..04b1d64 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -99,7 +99,6 @@ LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionL LZ4LIB_API int LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize); LZ4LIB_API int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, const char* src, char* dst, int srcSize, int maxDstSize); -LZ4LIB_API int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int* srcSizePtr, int maxDstSize); LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize); @@ -114,25 +113,23 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in Then, use LZ4_compress_HC_continue() to compress each successive block. Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression. - 'dst' buffer should be sized to handle worst case scenarios, using LZ4_compressBound(), to ensure operation success. + 'dst' buffer should be sized to handle worst case scenarios (see LZ4_compressBound()), to ensure operation success. + Because in case of failure, the API does not guarantee context recovery, and context will have to be reset. + If `dst` buffer budget cannot be >= LZ4_compressBound(), consider using LZ4_compress_HC_continue_destSize() instead. - If, for any reason, previous data blocks can't be preserved unmodified in memory during next compression block, - you must save it to a safer memory space, using LZ4_saveDictHC(). + If, for any reason, previous data block can't be preserved unmodified in memory for next compression block, + you can save it to a more stable memory space, using LZ4_saveDictHC(). Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer'. */ -/*-****************************************** - * !!!!! STATIC LINKING ONLY !!!!! - *******************************************/ - /*-************************************* * PRIVATE DEFINITIONS : * Do not use these definitions. * They are exposed to allow static allocation of `LZ4_streamHC_t`. * Using these definitions makes the code vulnerable to potential API break when upgrading LZ4 **************************************/ -#define LZ4HC_DICTIONARY_LOGSIZE 17 +#define LZ4HC_DICTIONARY_LOGSIZE 17 /* because of btopt, hc would only need 16 */ #define LZ4HC_MAXD (1< /* fgets, sscanf */ #include /* strcmp */ #include /* clock_t, clock, CLOCKS_PER_SEC */ +#define LZ4_HC_STATIC_LINKING_ONLY #include "lz4hc.h" #define XXH_STATIC_LINKING_ONLY #include "xxhash.h" @@ -353,11 +354,11 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_DISPLAYTEST; { int srcSize = blockSize; int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7; - char endCheck = FUZ_rand(&randState) & 255; - void * ctx = LZ4_createHC(block); + char const endCheck = FUZ_rand(&randState) & 255; + void* ctx = LZ4_createHC(block); FUZ_CHECKTEST(ctx==NULL, "LZ4_createHC() allocation failed"); compressedBuffer[targetSize] = endCheck; - ret = LZ4_compressHC_destSize(ctx, block, compressedBuffer, &srcSize, targetSize, compressionLevel); + ret = LZ4_compress_HC_destSize(ctx, block, compressedBuffer, &srcSize, targetSize, compressionLevel); LZ4_freeHC(ctx); FUZ_CHECKTEST(ret > targetSize, "LZ4_compressHC_destSize() result larger than dst buffer !"); FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compressHC_destSize() overwrite dst buffer !"); -- cgit v0.12