Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev

3 files changed, 25 insertions, 49 deletions

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index ae245b5..5d4ea3e 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -98,7 +98,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
 {
     MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
     MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
-    hc4->nextToUpdate = hc4->nextToUpdateBT = 64 KB;
+    hc4->nextToUpdate = 64 KB;
     hc4->base = start - 64 KB;
     hc4->end = start;
     hc4->dictBase = start - 64 KB;
@@ -486,6 +486,14 @@ _Search3:
     return (int) (((char*)op)-dest);
 }
 
+static int LZ4HC_getSearchNum(int compressionLevel)
+{
+    switch (compressionLevel) {
+        default: return 0; /* unused */
+        case 11: return 128; 
+        case 12: return 1<<10; 
+    }
+}
 
 static int LZ4HC_compress_generic (
     LZ4HC_CCtx_internal* const ctx,
@@ -501,9 +509,9 @@ static int LZ4HC_compress_generic (
     if (compressionLevel > 9) {
         switch (compressionLevel) {
             case 10: return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (16-1), limit);
-            case 11: ctx->searchNum = 128;  return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 128, 0);
+            case 11: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 128, 0);
             default:
-            case 12: ctx->searchNum = 1<<10; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM, 1);
+            case 12: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM, 1);
         }
     }
     return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (compressionLevel-1), limit);
@@ -554,6 +562,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
     LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET);   /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
     LZ4_streamHCPtr->internal_donotuse.compressionLevel = (unsigned)compressionLevel;
+    LZ4_streamHCPtr->internal_donotuse.searchNum = LZ4HC_getSearchNum(compressionLevel);
 }
 
 int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize)
@@ -564,8 +573,11 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
         dictSize = 64 KB;
     }
     LZ4HC_init (ctxPtr, (const BYTE*)dictionary);
-    if (dictSize >= 4) LZ4HC_Insert (ctxPtr, (const BYTE*)dictionary +(dictSize-3));
     ctxPtr->end = (const BYTE*)dictionary + dictSize;
+    if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN)
+        LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS);
+    else
+        if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);
     return dictSize;
 }
 
@@ -585,7 +597,7 @@ static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBl
     ctxPtr->dictBase  = ctxPtr->base;
     ctxPtr->base = newBlock - ctxPtr->dictLimit;
     ctxPtr->end  = newBlock;
-    ctxPtr->nextToUpdate = ctxPtr->nextToUpdateBT = ctxPtr->dictLimit;   /* match referencing will resume from there */
+    ctxPtr->nextToUpdate = ctxPtr->dictLimit;   /* match referencing will resume from there */
 }
 
 static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr,
@@ -645,7 +657,6 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS
         streamPtr->dictLimit = endIndex - dictSize;
         streamPtr->lowLimit = endIndex - dictSize;
         if (streamPtr->nextToUpdate < streamPtr->dictLimit) streamPtr->nextToUpdate = streamPtr->dictLimit;
-        if (streamPtr->nextToUpdateBT < streamPtr->dictLimit) streamPtr->nextToUpdateBT = streamPtr->dictLimit;
     }
     return dictSize;
 }
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index ad731d9..1036fd0 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -154,7 +154,6 @@ typedef struct
     uint32_t   dictLimit;        /* below that point, need extDict */
     uint32_t   lowLimit;         /* below that point, no more dict */
     uint32_t   nextToUpdate;     /* index from which to continue dictionary update */
-    uint32_t   nextToUpdateBT;   /* index from which to continue binary tree update */
     uint32_t   searchNum;        /* only for optimal parser */
     uint32_t   compressionLevel;
 } LZ4HC_CCtx_internal;
@@ -172,7 +171,6 @@ typedef struct
     unsigned int   dictLimit;        /* below that point, need extDict */
     unsigned int   lowLimit;         /* below that point, no more dict */
     unsigned int   nextToUpdate;     /* index from which to continue dictionary update */
-    unsigned int   nextToUpdateBT;     /* index from which to continue binary tree update */
     unsigned int   searchNum;        /* only for optimal parser */
     unsigned int   compressionLevel;
 } LZ4HC_CCtx_internal;
diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 94b38e0..d1913fe 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -33,11 +33,6 @@
        - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
 */
 
-#define LZ4_LOG_PARSER(fmt, ...)  //printf(fmt, __VA_ARGS__)
-#define LZ4_LOG_PRICE(fmt, ...)   //printf(fmt, __VA_ARGS__)
-#define LZ4_LOG_ENCODE(fmt, ...)  //printf(fmt, __VA_ARGS__)
-
-
 #define LZ4_OPT_NUM   (1<<12)
 
 
@@ -133,7 +128,10 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches (
         if (matchLength > best_mlen) {
             best_mlen = matchLength;
             if (matches) {
-                matches[mnum].off = (int)(ip - match);
+                if (matchIndex >= dictLimit)
+                    matches[mnum].off = (int)(ip - match);
+                else
+                    matches[mnum].off = (int)(ip - (base + matchIndex)); /* virtual matchpos */
                 matches[mnum].len = (int)matchLength;
                 mnum++;
             }
@@ -173,8 +171,7 @@ FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* cons
 {
     const BYTE* const base = ctx->base;
     const U32 target = (U32)(ip - base);
-    U32 idx = ctx->nextToUpdateBT;
-
+    U32 idx = ctx->nextToUpdate;
     while(idx < target)
         idx += LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, 8, NULL, NULL);
 }
@@ -187,10 +184,10 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches (
                         size_t best_mlen, LZ4HC_match_t* matches, const int fullUpdate)
 {
     int mnum = 0;
-    if (ip < ctx->base + ctx->nextToUpdateBT) return 0;   /* skipped area */
+    if (ip < ctx->base + ctx->nextToUpdate) return 0;   /* skipped area */
     if (fullUpdate) LZ4HC_updateBinTree(ctx, ip, iHighLimit);
     best_mlen = LZ4HC_BinTree_InsertAndGetAllMatches(ctx, ip, iHighLimit, best_mlen, matches, &mnum);
-    ctx->nextToUpdateBT = (U32)(ip - ctx->base + best_mlen);
+    ctx->nextToUpdate = (U32)(ip - ctx->base + best_mlen);
     return mnum;
 }
 
@@ -202,7 +199,6 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches (
     opt[pos].off = (int)offset;                        \
     opt[pos].litlen = (int)litlen;                     \
     opt[pos].price = (int)price;                       \
-    LZ4_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-source), pos, last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \
 }
 
 
@@ -219,7 +215,7 @@ static int LZ4HC_compress_optimal (
 {
     LZ4HC_optimal_t opt[LZ4_OPT_NUM + 1];
     LZ4HC_match_t matches[LZ4_OPT_NUM + 1];
-    const BYTE *inr;
+    const BYTE *inr = NULL;
     size_t res, cur, cur2;
     size_t i, llen, litlen, mlen, best_mlen, price, offset, best_off, match_num, last_pos;
 
@@ -242,7 +238,6 @@ static int LZ4HC_compress_optimal (
         llen = ip - anchor;
         match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate);
         if (!match_num) { ip++; continue; }
-        LZ4_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-source), match_num, last_pos);
 
         if ((size_t)matches[match_num-1].len > sufficient_len) {
             best_mlen = matches[match_num-1].len;
@@ -256,7 +251,6 @@ static int LZ4HC_compress_optimal (
         for (i = 0; i < match_num; i++) {
            mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH;
            best_mlen = (matches[i].len < LZ4_OPT_NUM) ? matches[i].len : LZ4_OPT_NUM;
-           LZ4_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-source), matches[i].len, matches[i].off, best_mlen, last_pos);
            while (mlen <= best_mlen) {
                 litlen = 0;
                 price = LZ4HC_sequencePrice(llen + litlen, mlen) - LZ4HC_literalsPrice(llen);
@@ -276,29 +270,22 @@ static int LZ4HC_compress_optimal (
                 litlen = opt[cur-1].litlen + 1;
                 if (cur != litlen) {
                     price = opt[cur - litlen].price + LZ4HC_literalsPrice(litlen);
-                    LZ4_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen);
                 } else {
                     price = LZ4HC_literalsPrice(llen + litlen) - LZ4HC_literalsPrice(llen);
-                    LZ4_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen);
                 }
             } else {
                 litlen = 1;
                 price = opt[cur - 1].price + LZ4HC_literalsPrice(litlen);
-                LZ4_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-source), price, cur, litlen, LZ4HC_literalsPrice(litlen));
             }
 
             mlen = 1;
             best_mlen = 0;
-            LZ4_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-source), price, cur, opt[cur].price);
             if (cur > last_pos || price < (size_t)opt[cur].price)
                 SET_PRICE(cur, mlen, best_mlen, litlen, price);
 
             if (cur == last_pos || inr >= mflimit) break;
-            LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen);
 
             match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches, fullUpdate);
-            LZ4_LOG_PARSER("%d: LZ4HC_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num);
-
             if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) {
                 best_mlen = matches[match_num-1].len;
                 best_off = matches[match_num-1].off;
@@ -311,7 +298,6 @@ static int LZ4HC_compress_optimal (
                 mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH;
                 cur2 = cur;
                 best_mlen = (cur2 + matches[i].len < LZ4_OPT_NUM) ? (size_t)matches[i].len : LZ4_OPT_NUM - cur2;
-                LZ4_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-source), cur, cur2, matches[i].len, matches[i].off, best_mlen, last_pos);
 
                 while (mlen <= best_mlen) {
                     if (opt[cur2].mlen == 1) {
@@ -326,7 +312,6 @@ static int LZ4HC_compress_optimal (
                         price = opt[cur2].price + LZ4HC_sequencePrice(litlen, mlen);
                     }
 
-                    LZ4_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price);
                     if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) { // || (((int)price == opt[cur2 + mlen].price) && (opt[cur2 + mlen-1].mlen == 1))) {
                         SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price);
                     }
@@ -340,12 +325,6 @@ static int LZ4HC_compress_optimal (
         cur = last_pos - best_mlen;
 
 encode: /* cur, last_pos, best_mlen, best_off have to be set */
-        for (i = 1; i <= last_pos; i++) {
-            LZ4_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen);
-        }
-
-        LZ4_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d\n", (int)(ip-source+cur), cur, last_pos, best_mlen, best_off);
-
         opt[0].mlen = 1;
         while (1) {
             mlen = opt[cur].mlen;
@@ -358,26 +337,15 @@ encode: /* cur, last_pos, best_mlen, best_off have to be set */
             cur -= mlen;
         }
 
-        for (i = 0; i <= last_pos;) {
-            LZ4_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen);
-            i += opt[i].mlen;
-        }
-
         cur = 0;
         while (cur < last_pos) {
-            LZ4_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen);
             mlen = opt[cur].mlen;
             if (mlen == 1) { ip++; cur++; continue; }
             offset = opt[cur].off;
             cur += mlen;
 
-            LZ4_LOG_ENCODE("%d: ENCODE literals=%d off=%d mlen=%d ", (int)(ip-(const BYTE*)source), (int)(ip-anchor), (int)(offset), (int)mlen);
             res = LZ4HC_encodeSequence(&ip, &op, &anchor, (int)mlen, ip - offset, limit, oend);
-            LZ4_LOG_ENCODE("out=%d\n", (int)((char*)op - dest));
-
             if (res) return 0;
-
-            LZ4_LOG_PARSER("%d: offset=%d\n", (int)(ip-source), offset);
         }
     }
 
@@ -386,7 +354,6 @@ encode: /* cur, last_pos, best_mlen, best_off have to be set */
         if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0;  /* Check output limit */
         if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun > 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
         else *op++ = (BYTE)(lastRun<<ML_BITS);
-        LZ4_LOG_ENCODE("%d: ENCODE_LAST literals=%d out=%d\n", (int)(ip-(const BYTE*)source), (int)(iend-anchor), (int)((char*)op-dest));
         memcpy(op, anchor, iend - anchor);
         op += iend-anchor;
     }