From 838ed341f94aa68c0bce8ee181e33a87274d2202 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 9 Dec 2016 17:16:35 +0100 Subject: fullUpdate as a parameter --- lib/lz4hc.c | 11 +++++++---- lib/lz4hc.h | 2 ++ lib/lz4opt.h | 23 ++++++++++++----------- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index d3578b9..e1eadf7 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -98,7 +98,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) { MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); - hc4->nextToUpdate = 64 KB; + hc4->nextToUpdate = hc4->nextToUpdateBT = 64 KB; hc4->base = start - 64 KB; hc4->end = start; hc4->dictBase = start - 64 KB; @@ -501,9 +501,9 @@ static int LZ4HC_compress_generic ( if (compressionLevel > 9) { switch (compressionLevel) { case 10: return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (16-1), limit); - case 11: ctx->searchNum = 256; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 256); + case 11: ctx->searchNum = 128; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 128, 0); default: - case 12: ctx->searchNum = 1<<14; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM); + case 12: ctx->searchNum = 1<<10; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM, 1); } } return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (compressionLevel-1), limit); @@ -575,13 +575,15 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) { if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ + LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS); + /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ ctxPtr->lowLimit = ctxPtr->dictLimit; ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); ctxPtr->dictBase = ctxPtr->base; ctxPtr->base = newBlock - ctxPtr->dictLimit; ctxPtr->end = newBlock; - ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ + ctxPtr->nextToUpdate = ctxPtr->nextToUpdateBT = ctxPtr->dictLimit; /* match referencing will resume from there */ } static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, @@ -641,6 +643,7 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS streamPtr->dictLimit = endIndex - dictSize; streamPtr->lowLimit = endIndex - dictSize; if (streamPtr->nextToUpdate < streamPtr->dictLimit) streamPtr->nextToUpdate = streamPtr->dictLimit; + if (streamPtr->nextToUpdateBT < streamPtr->dictLimit) streamPtr->nextToUpdateBT = streamPtr->dictLimit; } return dictSize; } diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 59d912e..a3736f8 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -153,6 +153,7 @@ typedef struct uint32_t dictLimit; /* below that point, need extDict */ uint32_t lowLimit; /* below that point, no more dict */ uint32_t nextToUpdate; /* index from which to continue dictionary update */ + uint32_t nextToUpdateBT; /* index from which to continue binary tree update */ uint32_t searchNum; /* only for optimal parser */ uint32_t compressionLevel; } LZ4HC_CCtx_internal; @@ -170,6 +171,7 @@ typedef struct unsigned int dictLimit; /* below that point, need extDict */ unsigned int lowLimit; /* below that point, no more dict */ unsigned int nextToUpdate; /* index from which to continue dictionary update */ + unsigned int nextToUpdateBT; /* index from which to continue binary tree update */ unsigned int searchNum; /* only for optimal parser */ unsigned int compressionLevel; } LZ4HC_CCtx_internal; diff --git a/lib/lz4opt.h b/lib/lz4opt.h index c466677..0844f8e 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -106,7 +106,7 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( if (ip + MINMATCH > iHighLimit) return 1; - /* First Match */ + /* HC4 match finder */ HashPos = &HashTable[LZ4HC_hashPtr(ip)]; matchIndex = *HashPos; *HashPos = current; @@ -162,7 +162,7 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( *ptr0 = (U16)-1; *ptr1 = (U16)-1; if (matchNum) *matchNum = mnum; - // if (best_mlen > 8) return best_mlen-8; + /* if (best_mlen > 8) return best_mlen-8; */ if (!matchNum) return 1; return 1; } @@ -172,24 +172,24 @@ FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* cons { const BYTE* const base = ctx->base; const U32 target = (U32)(ip - base); - U32 idx = ctx->nextToUpdate; + U32 idx = ctx->nextToUpdateBT; while(idx < target) idx += LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, 8, NULL, NULL); -} +} /** Tree updater, providing best match */ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( LZ4HC_CCtx_internal* ctx, const BYTE* const ip, const BYTE* const iHighLimit, - size_t best_mlen, LZ4HC_match_t* matches) + size_t best_mlen, LZ4HC_match_t* matches, const int fullUpdate) { int mnum = 0; - if (ip < ctx->base + ctx->nextToUpdate) return 0; /* skipped area */ - LZ4HC_updateBinTree(ctx, ip, iHighLimit); + if (ip < ctx->base + ctx->nextToUpdateBT) return 0; /* skipped area */ + if (fullUpdate) LZ4HC_updateBinTree(ctx, ip, iHighLimit); best_mlen = LZ4HC_BinTree_InsertAndGetAllMatches(ctx, ip, iHighLimit, best_mlen, matches, &mnum); - ctx->nextToUpdate = (U32)(ip - ctx->base) + best_mlen; + ctx->nextToUpdateBT = (U32)(ip - ctx->base) + best_mlen; return mnum; } @@ -212,7 +212,8 @@ static int LZ4HC_compress_optimal ( int inputSize, int maxOutputSize, limitedOutput_directive limit, - const size_t sufficient_len + const size_t sufficient_len, + const int fullUpdate ) { LZ4HC_optimal_t opt[LZ4_OPT_NUM + 1]; @@ -238,7 +239,7 @@ static int LZ4HC_compress_optimal ( memset(opt, 0, sizeof(LZ4HC_optimal_t)); last_pos = 0; llen = ip - anchor; - match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches); + match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate); if (!match_num) { ip++; continue; } LZ4_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-source), match_num, last_pos); @@ -294,7 +295,7 @@ static int LZ4HC_compress_optimal ( if (cur == last_pos || inr >= mflimit) break; LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); - match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches); + match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches, fullUpdate); LZ4_LOG_PARSER("%d: LZ4HC_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) { -- cgit v0.12