From 1148173c5dd1ad9b672c63fd0da110e3c2d66274 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 26 Apr 2018 13:01:59 -0700 Subject: introduced ability to parse for decompression speed triggered through an enum. Now, it's still necessary to properly expose this capability all the way up to the cli. --- lib/lz4hc.c | 53 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 4126ef8..3593da7 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -199,6 +199,7 @@ LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern) } typedef enum { rep_untested, rep_not, rep_confirmed } repeat_state_e; +typedef enum { favorCompressionRatio=0, favorDecompressionSpeed } HCfavor_e; LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( @@ -211,7 +212,8 @@ LZ4HC_InsertAndGetWiderMatch ( const BYTE** startpos, const int maxNbAttempts, const int patternAnalysis, - const dictCtx_directive dict) + const dictCtx_directive dict, + const HCfavor_e favorDecSpeed) { U16* const chainTable = hc4->chainTable; U32* const HashTable = hc4->hashTable; @@ -240,7 +242,10 @@ LZ4HC_InsertAndGetWiderMatch ( while ((matchIndex>=lowLimit) && (nbAttempts)) { DEBUGLOG(7, "remaining attempts : %i", nbAttempts); nbAttempts--; - if (matchIndex >= dictLimit) { + assert(matchIndex < ipIndex); + if (favorDecSpeed && (ipIndex - matchIndex < 8)) { + /* do nothing */ + } else if (matchIndex >= dictLimit) { const BYTE* const matchPtr = base + matchIndex; assert(longest >= 1); if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - delta + longest - 1)) { @@ -326,14 +331,12 @@ LZ4HC_InsertAndGetWiderMatch ( } } - { - U32 const nextOffset = DELTANEXTU16(dictCtx->chainTable, dictMatchIndex); + { U32 const nextOffset = DELTANEXTU16(dictCtx->chainTable, dictMatchIndex); dictMatchIndex -= nextOffset; matchIndex -= nextOffset; } } } - return longest; } @@ -349,7 +352,7 @@ int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index tabl /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos), * but this won't be the case here, as we define iLowLimit==ip, * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */ - return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis, dict); + return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis, dict, favorCompressionRatio); } @@ -484,7 +487,7 @@ _Search2: if (ip+ml <= mflimit) ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2, - maxNbAttempts, patternAnalysis, dict); + maxNbAttempts, patternAnalysis, dict, favorCompressionRatio); else ml2 = ml; @@ -531,7 +534,7 @@ _Search3: if (start2 + ml2 <= mflimit) ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, - maxNbAttempts, patternAnalysis, dict); + maxNbAttempts, patternAnalysis, dict, favorCompressionRatio); else ml3 = ml2; @@ -651,12 +654,14 @@ _dest_overflow: return 0; } + static int LZ4HC_compress_optimal( LZ4HC_CCtx_internal* ctx, const char* const source, char* dst, int* srcSizePtr, int dstCapacity, int const nbSearches, size_t sufficient_len, const limitedOutput_directive limit, int const fullUpdate, - const dictCtx_directive dict); + const dictCtx_directive dict, + HCfavor_e favorDecSpeed); LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal ( @@ -711,7 +716,9 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal ( return LZ4HC_compress_optimal(ctx, src, dst, srcSizePtr, dstCapacity, cParam.nbSearches, cParam.targetLength, limit, - cLevel == LZ4HC_CLEVEL_MAX, dict); /* ultra mode */ + cLevel == LZ4HC_CLEVEL_MAX, /* ultra mode */ + dict, + favorDecompressionSpeed); } } @@ -1082,23 +1089,26 @@ LZ4_FORCE_INLINE LZ4HC_match_t LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx, const BYTE* ip, const BYTE* const iHighLimit, int minLen, int nbSearches, - const dictCtx_directive dict) + const dictCtx_directive dict, + const HCfavor_e favorDecSpeed) { LZ4HC_match_t match = { 0 , 0 }; const BYTE* matchPtr = NULL; /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos), * but this won't be the case here, as we define iLowLimit==ip, * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */ - int const matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, + int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, &matchPtr, &ip, - nbSearches, 1 /* patternAnalysis */, dict); + nbSearches, 1 /* patternAnalysis */, dict, favorDecSpeed); if (matchLength <= minLen) return match; + if (favorDecSpeed) { + if ((matchLength>18) & (matchLength<=36)) matchLength=18; /* favor shortcut */ + } match.len = matchLength; match.off = (int)(ip-matchPtr); return match; } - static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, const char* const source, @@ -1109,7 +1119,8 @@ static int LZ4HC_compress_optimal ( size_t sufficient_len, const limitedOutput_directive limit, int const fullUpdate, - const dictCtx_directive dict + const dictCtx_directive dict, + HCfavor_e favorDecSpeed ) { #define TRAILING_LITERALS 3 @@ -1125,6 +1136,7 @@ static int LZ4HC_compress_optimal ( BYTE* oend = op + dstCapacity; /* init */ + favorDecSpeed = favorCompressionRatio; DEBUGLOG(5, "LZ4HC_compress_optimal"); *srcSizePtr = 0; if (limit == limitedDestSize) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ @@ -1137,7 +1149,7 @@ static int LZ4HC_compress_optimal ( int best_mlen, best_off; int cur, last_match_pos = 0; - LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches, dict); + LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches, dict, favorDecSpeed); if (firstMatch.len==0) { ip++; continue; } if ((size_t)firstMatch.len > sufficient_len) { @@ -1207,10 +1219,10 @@ static int LZ4HC_compress_optimal ( DEBUGLOG(7, "search at rPos:%u", cur); if (fullUpdate) - newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches, dict); + newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches, dict, favorDecSpeed); else /* only test matches of minimum length; slightly faster, but misses a few bytes */ - newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches, dict); + newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches, dict, favorDecSpeed); if (!newMatch.len) continue; if ( ((size_t)newMatch.len > sufficient_len) @@ -1258,7 +1270,10 @@ static int LZ4HC_compress_optimal ( price = opt[cur].price + LZ4HC_sequencePrice(0, ml); } - if (pos > last_match_pos+TRAILING_LITERALS || price <= opt[pos].price) { + assert(opt[pos].price > 1); + assert((U32)favorDecSpeed <= 1); + if (pos > last_match_pos+TRAILING_LITERALS + || price <= opt[pos].price - (int)favorDecSpeed) { DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i)", pos, price, ml); assert(pos < LZ4_OPT_NUM); -- cgit v0.12 From 3792d00168edd060c58ceaecffb97d43dab27094 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 26 Apr 2018 15:18:44 -0700 Subject: favorDecSpeed feature can be triggered from lz4frame and lz4hc. --- lib/lz4frame.c | 3 +++ lib/lz4frame.h | 9 +++++---- lib/lz4hc.c | 8 ++++++-- lib/lz4hc.h | 18 +++++++++++++----- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index b616463..06a0f7b 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -612,6 +612,9 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, /* frame init only for blockLinked : blockIndependent will be init at each block */ LZ4F_applyCDict(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel); } + if (preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) { + LZ4_favorDecompressionSpeed(cctxPtr->lz4CtxPtr, preferencesPtr->favorDecSpeed); + } /* Magic Number */ LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER); diff --git a/lib/lz4frame.h b/lib/lz4frame.h index bd715bd..fb434ff 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -173,13 +173,14 @@ typedef struct { /*! LZ4F_preferences_t : * makes it possible to supply detailed compression parameters to the stream interface. - * It's not required to set all fields, as long as the structure was initially memset() to zero. + * Structure is presumed initially memset() to zero, representing default settings. * All reserved fields must be set to zero. */ typedef struct { LZ4F_frameInfo_t frameInfo; - int compressionLevel; /* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */ - unsigned autoFlush; /* 1 == always flush, to reduce usage of internal buffers */ - unsigned reserved[4]; /* must be zero for forward compatibility */ + int compressionLevel; /* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */ + unsigned autoFlush; /* 1: always flush, to reduce usage of internal buffers */ + unsigned favorDecSpeed; /* 1: parser favors decompression speed vs compression ratio. Only works for high compression modes (>= LZ4LZ4HC_CLEVEL_OPT_MIN) */ /* >= v1.8.2 */ + unsigned reserved[3]; /* must be zero for forward compatibility */ } LZ4F_preferences_t; LZ4FLIB_API int LZ4F_compressionLevel_max(void); diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 3593da7..b90d60b 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -876,6 +876,11 @@ void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLev LZ4_streamHCPtr->internal_donotuse.compressionLevel = compressionLevel; } +void LZ4_favorDecompressionSpeed(LZ4_streamHC_t* LZ4_streamHCPtr, int favor) +{ + LZ4_streamHCPtr->internal_donotuse.favorDecSpeed = (favor!=0); +} + int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) { LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; @@ -1120,7 +1125,7 @@ static int LZ4HC_compress_optimal ( const limitedOutput_directive limit, int const fullUpdate, const dictCtx_directive dict, - HCfavor_e favorDecSpeed + const HCfavor_e favorDecSpeed ) { #define TRAILING_LITERALS 3 @@ -1136,7 +1141,6 @@ static int LZ4HC_compress_optimal ( BYTE* oend = op + dstCapacity; /* init */ - favorDecSpeed = favorCompressionRatio; DEBUGLOG(5, "LZ4HC_compress_optimal"); *srcSizePtr = 0; if (limit == limitedDestSize) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 28e2528..bb5e073 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -152,7 +152,8 @@ struct LZ4HC_CCtx_internal uint32_t dictLimit; /* below that point, need extDict */ uint32_t lowLimit; /* below that point, no more dict */ uint32_t nextToUpdate; /* index from which to continue dictionary update */ - int compressionLevel; + short compressionLevel; + short favorDecSpeed; const LZ4HC_CCtx_internal* dictCtx; }; @@ -169,7 +170,8 @@ struct LZ4HC_CCtx_internal unsigned int dictLimit; /* below that point, need extDict */ unsigned int lowLimit; /* below that point, no more dict */ unsigned int nextToUpdate; /* index from which to continue dictionary update */ - int compressionLevel; + short compressionLevel; + short favorDecSpeed; const LZ4HC_CCtx_internal* dictCtx; }; @@ -253,9 +255,9 @@ LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") LZ4LIB_API int LZ4_resetStr * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src` */ int LZ4_compress_HC_destSize(void* LZ4HC_Data, - const char* src, char* dst, - int* srcSizePtr, int targetDstSize, - int compressionLevel); + const char* src, char* dst, + int* srcSizePtr, int targetDstSize, + int compressionLevel); /*! LZ4_compress_HC_continue_destSize() : v1.8.0 (experimental) * Similar as LZ4_compress_HC_continue(), @@ -275,6 +277,12 @@ int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr, */ void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); +/*! LZ4_favorDecompressionSpeed() : v1.8.2 (experimental) + * Parser will select decisions favoring decompression over compression ratio. + * Only work at highest compression settings (level >= LZ4HC_CLEVEL_OPT_MIN) + */ +void LZ4_favorDecompressionSpeed(LZ4_streamHC_t* LZ4_streamHCPtr, int favor); + /*! LZ4_resetStreamHC_fast() : * When an LZ4_streamHC_t is known to be in a internally coherent state, * it can often be prepared for a new compression with almost no work, only -- cgit v0.12 From 5c7d3812d90aeaf072d14f6b5d935711da6f14c7 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 26 Apr 2018 15:49:32 -0700 Subject: fasterDecSpeed can be triggered from cli with --favor-decSpeed --- lib/lz4frame.c | 2 +- lib/lz4hc.c | 2 +- programs/lz4cli.c | 2 ++ programs/lz4io.c | 8 ++++++++ programs/lz4io.h | 7 ++++++- 5 files changed, 18 insertions(+), 3 deletions(-) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 06a0f7b..9d88644 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -613,7 +613,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, LZ4F_applyCDict(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel); } if (preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) { - LZ4_favorDecompressionSpeed(cctxPtr->lz4CtxPtr, preferencesPtr->favorDecSpeed); + LZ4_favorDecompressionSpeed(cctxPtr->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed); } /* Magic Number */ diff --git a/lib/lz4hc.c b/lib/lz4hc.c index b90d60b..39ab5fb 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -718,7 +718,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal ( cParam.nbSearches, cParam.targetLength, limit, cLevel == LZ4HC_CLEVEL_MAX, /* ultra mode */ dict, - favorDecompressionSpeed); + ctx->favorDecSpeed); } } diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 42392eb..ba519b4 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -140,6 +140,7 @@ static int usage_advanced(const char* exeName) DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled) \n"); DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n"); DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); + DISPLAY( "--favor-decSpeed: compressed files decompress faster, but are less compressed \n"); DISPLAY( "Benchmark arguments : \n"); DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n"); DISPLAY( " -e# : test all compression levels from -bX to # (default : 1)\n"); @@ -355,6 +356,7 @@ int main(int argc, const char** argv) if (!strcmp(argument, "--no-content-size")) { LZ4IO_setContentSize(0); continue; } if (!strcmp(argument, "--sparse")) { LZ4IO_setSparseFile(2); continue; } if (!strcmp(argument, "--no-sparse")) { LZ4IO_setSparseFile(0); continue; } + if (!strcmp(argument, "--favor-decSpeed")) { LZ4IO_favorDecSpeed(1); continue; } if (!strcmp(argument, "--verbose")) { displayLevel++; continue; } if (!strcmp(argument, "--quiet")) { if (displayLevel) displayLevel--; continue; } if (!strcmp(argument, "--version")) { DISPLAY(WELCOME_MESSAGE); return 0; } diff --git a/programs/lz4io.c b/programs/lz4io.c index ccf4fa1..b52c1f3 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -116,6 +116,7 @@ static int g_blockIndependence = 1; static int g_sparseFileSupport = 1; static int g_contentSizeFlag = 0; static int g_useDictionary = 0; +static unsigned g_favorDecSpeed = 0; static const char* g_dictionaryFilename = NULL; @@ -221,6 +222,12 @@ int LZ4IO_setContentSize(int enable) return g_contentSizeFlag; } +/* Default setting : 0 (disabled) */ +void LZ4IO_favorDecSpeed(int favor) +{ + g_favorDecSpeed = (favor!=0); +} + static U32 g_removeSrcFile = 0; void LZ4IO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); } @@ -548,6 +555,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)g_blockSizeId; prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)g_blockChecksum; prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)g_streamChecksum; + prefs.favorDecSpeed = g_favorDecSpeed; if (g_contentSizeFlag) { U64 const fileSize = UTIL_getFileSize(srcFileName); prefs.frameInfo.contentSize = fileSize; /* == 0 if input == stdin */ diff --git a/programs/lz4io.h b/programs/lz4io.h index b21b8b6..22c5e3e 100644 --- a/programs/lz4io.h +++ b/programs/lz4io.h @@ -94,10 +94,15 @@ int LZ4IO_setNotificationLevel(int level); /* Default setting : 0 (disabled) */ int LZ4IO_setSparseFile(int enable); -/* Default setting : 0 (disabled) */ +/* Default setting : 0 == no content size present in frame header */ int LZ4IO_setContentSize(int enable); +/* Default setting : 0 == src file preserved */ void LZ4IO_setRemoveSrcFile(unsigned flag); +/* Default setting : 0 == favor compression ratio + * Note : 1 only works for high compression levels (10+) */ +void LZ4IO_favorDecSpeed(int favor); + #endif /* LZ4IO_H_237902873 */ -- cgit v0.12 From 0fb3a3b199ed0a92a402c5979c514a329b85462a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 26 Apr 2018 17:02:20 -0700 Subject: fixed a number of minor cast warnings --- examples/frameCompress.c | 3 ++- lib/lz4frame.c | 2 +- lib/lz4hc.c | 9 ++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/frameCompress.c b/examples/frameCompress.c index 972f716..9bfea48 100644 --- a/examples/frameCompress.c +++ b/examples/frameCompress.c @@ -21,7 +21,8 @@ static const LZ4F_preferences_t kPrefs = { 0 /* unknown content size */, 0 /* no dictID */ , LZ4F_noBlockChecksum }, 0, /* compression level; 0 == default */ 0, /* autoflush */ - { 0, 0, 0, 0 }, /* reserved, must be set to 0 */ + 0, /* favor decompression speed */ + { 0, 0, 0 }, /* reserved, must be set to 0 */ }; diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 9d88644..0338266 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -613,7 +613,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, LZ4F_applyCDict(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel); } if (preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) { - LZ4_favorDecompressionSpeed(cctxPtr->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed); + LZ4_favorDecompressionSpeed((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed); } /* Magic Number */ diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 39ab5fb..35eac1a 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -708,6 +708,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal ( assert(cLevel >= 0); assert(cLevel <= LZ4HC_CLEVEL_MAX); { cParams_t const cParam = clTable[cLevel]; + HCfavor_e const favor = ctx->favorDecSpeed ? favorDecompressionSpeed : favorCompressionRatio; if (cParam.strat == lz4hc) return LZ4HC_compress_hashChain(ctx, src, dst, srcSizePtr, dstCapacity, @@ -717,8 +718,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal ( src, dst, srcSizePtr, dstCapacity, cParam.nbSearches, cParam.targetLength, limit, cLevel == LZ4HC_CLEVEL_MAX, /* ultra mode */ - dict, - ctx->favorDecSpeed); + dict, favor); } } @@ -756,7 +756,7 @@ static int LZ4HC_compress_generic_dictCtx ( } else if (position == 0 && *srcSizePtr > 4 KB) { memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal)); LZ4HC_setExternalDict(ctx, (const BYTE *)src); - ctx->compressionLevel = cLevel; + ctx->compressionLevel = (short)cLevel; return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); } else { return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx); @@ -873,7 +873,7 @@ void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLev { if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT; if (compressionLevel > LZ4HC_CLEVEL_MAX) compressionLevel = LZ4HC_CLEVEL_MAX; - LZ4_streamHCPtr->internal_donotuse.compressionLevel = compressionLevel; + LZ4_streamHCPtr->internal_donotuse.compressionLevel = (short)compressionLevel; } void LZ4_favorDecompressionSpeed(LZ4_streamHC_t* LZ4_streamHCPtr, int favor) @@ -1274,7 +1274,6 @@ static int LZ4HC_compress_optimal ( price = opt[cur].price + LZ4HC_sequencePrice(0, ml); } - assert(opt[pos].price > 1); assert((U32)favorDecSpeed <= 1); if (pos > last_match_pos+TRAILING_LITERALS || price <= opt[pos].price - (int)favorDecSpeed) { -- cgit v0.12 From d294dd7fc684b1f46130358ff5c218e835082b16 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 27 Apr 2018 09:04:09 -0700 Subject: ensure favorDecSpeed is properly initialized also : - fix a potential malloc error - proper use of ALLOC macro inside lz4hc - update html API doc --- doc/lz4_manual.html | 12 +++++---- doc/lz4frame_manual.html | 63 +++++++++++++++++++++++++++++++++++++++++++++--- lib/lz4.c | 6 ++--- lib/lz4hc.c | 10 ++++---- 4 files changed, 74 insertions(+), 17 deletions(-) diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html index f8639fe..ddd2724 100644 --- a/doc/lz4_manual.html +++ b/doc/lz4_manual.html @@ -126,15 +126,17 @@ int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int src 
int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
 
This function is a bit faster than LZ4_decompress_safe(),
-but doesn't provide any security guarantee.
+but it may misbehave on malformed input because it doesn't perform full validation of compressed data.
     originalSize : is the uncompressed size to regenerate
                    Destination buffer must be already allocated, and its size must be >= 'originalSize' bytes.
     return : number of bytes read from source buffer (== compressed size).
              If the source stream is detected malformed, the function stops decoding and return a negative result.
-    note : This function respects memory boundaries for *properly formed* compressed data.
-           However, it does not provide any protection against malicious input.
-           It also doesn't know 'src' size, and implies it's >= compressed size.
-           Use this function in trusted environment **only**.
+    note : This function is only usable if the originalSize of uncompressed data is known in advance.
+           The caller should also check that all the compressed input has been consumed properly,
+           i.e. that the return value matches the size of the buffer with compressed input.
+           The function never writes past the output buffer.  However, since it doesn't know its 'src' size,
+           it may read past the intended input.  Also, because match offsets are not validated during decoding,
+           reads from 'src' may underflow.  Use this function in trusted environment **only**.
 

int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity);
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index 459bac8..53ea7eb 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -18,6 +18,7 @@
 
  • Compression
    • 
 
  • Decompression functions
    • 
 
  • Streaming decompression functions
    • 
+
  • Bulk processing dictionary API
    • 
 
 
    
 
    Introduction
    @@ -89,12 +90,13 @@
 
 
    typedef struct {
   LZ4F_frameInfo_t frameInfo;
-  int      compressionLevel;       /* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */
-  unsigned autoFlush;              /* 1 == always flush, to reduce usage of internal buffers */
-  unsigned reserved[4];            /* must be zero for forward compatibility */
+  int      compressionLevel;    /* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */
+  unsigned autoFlush;           /* 1: always flush, to reduce usage of internal buffers */
+  unsigned favorDecSpeed;       /* 1: parser favors decompression speed vs compression ratio. Only works for high compression modes (>= LZ4LZ4HC_CLEVEL_OPT_MIN) */  /* >= v1.8.2 */
+  unsigned reserved[3];         /* must be zero for forward compatibility */
 } LZ4F_preferences_t;
 
      makes it possible to supply detailed compression parameters to the stream interface.
-  It's not required to set all fields, as long as the structure was initially memset() to zero.
+  Structure is presumed initially memset() to zero, representing default settings.
   All reserved fields must be set to zero. 
 

    
 
@@ -293,5 +295,58 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
   and start a new one using same context resources. 
 

    
 
+
    typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes;
+

    
+
    Bulk processing dictionary API
    
+
+
    LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize);
+LZ4FLIB_STATIC_API void        LZ4F_freeCDict(LZ4F_CDict* CDict);
+
      When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
+  LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+  LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
+ `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict 
+

    
+
+
    LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict(
+    LZ4F_cctx* cctx,
+    void* dst, size_t dstCapacity,
+    const void* src, size_t srcSize,
+    const LZ4F_CDict* cdict,
+    const LZ4F_preferences_t* preferencesPtr);
+
      Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary.
+  cctx must point to a context created by LZ4F_createCompressionContext().
+  If cdict==NULL, compress without a dictionary.
+  dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
+  If this condition is not respected, function will fail (@return an errorCode).
+  The LZ4F_preferences_t structure is optional : you may provide NULL as argument,
+  but it's not recommended, as it's the only way to provide dictID in the frame header.
+ @return : number of bytes written into dstBuffer.
+           or an error code if it fails (can be tested using LZ4F_isError()) 
+

    
+
+
    LZ4FLIB_STATIC_API size_t LZ4F_compressBegin_usingCDict(
+    LZ4F_cctx* cctx,
+    void* dstBuffer, size_t dstCapacity,
+    const LZ4F_CDict* cdict,
+    const LZ4F_preferences_t* prefsPtr);
+
      Inits streaming dictionary compression, and writes the frame header into dstBuffer.
+  dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
+ `prefsPtr` is optional : you may provide NULL as argument,
+  however, it's the only way to provide dictID in the frame header.
+ @return : number of bytes written into dstBuffer for the header,
+           or an error code (which can be tested using LZ4F_isError()) 
+

    
+
+
    LZ4FLIB_STATIC_API size_t LZ4F_decompress_usingDict(
+    LZ4F_dctx* dctxPtr,
+    void* dstBuffer, size_t* dstSizePtr,
+    const void* srcBuffer, size_t* srcSizePtr,
+    const void* dict, size_t dictSize,
+    const LZ4F_decompressOptions_t* decompressOptionsPtr);
+
      Same as LZ4F_decompress(), using a predefined dictionary.
+  Dictionary is used "in place", without any preprocessing.
+  It must remain accessible throughout the entire frame decoding. 
+

    
+
 
 
diff --git a/lib/lz4.c b/lib/lz4.c
index eaabe5b..deaec7b 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -161,11 +161,11 @@
 *  Memory routines
 **************************************/
 #include    /* malloc, calloc, free */
-#define ALLOC(s) malloc(s)
+#define ALLOC(s)          malloc(s)
 #define ALLOC_AND_ZERO(s) calloc(1,s)
-#define FREEMEM        free
+#define FREEMEM(p)        free(p)
 #include    /* memset, memcpy */
-#define MEM_INIT       memset
+#define MEM_INIT(p,v,s)   memset((p),(v),(s))
 
 
 /*-************************************
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 35eac1a..1c7096b 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -805,7 +805,7 @@ int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int src
 int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
 {
 #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
-    LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
+    LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t));
 #else
     LZ4_streamHC_t state;
     LZ4_streamHC_t* const statePtr = &state;
@@ -834,10 +834,9 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i
 **************************************/
 /* allocation */
 LZ4_streamHC_t* LZ4_createStreamHC(void) {
-    LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
-    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
-    LZ4_streamHCPtr->internal_donotuse.base = NULL;
-    LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
+    LZ4_streamHC_t* const LZ4_streamHCPtr = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t));
+    if (LZ4_streamHCPtr==NULL) return NULL;
+    LZ4_resetStreamHC(LZ4_streamHCPtr, LZ4HC_CLEVEL_DEFAULT);
     return LZ4_streamHCPtr;
 }
 
@@ -857,6 +856,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
     LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
     LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
+    LZ4_streamHCPtr->internal_donotuse.favorDecSpeed = 0;
     LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
 }
 
-- 
cgit v0.12