Continuous streaming mode (automatic)

3 files changed, 63 insertions, 22 deletions

diff --git a/Makefile b/Makefile
index 11fa7b3..c80e02c 100644
--- a/Makefile
+++ b/Makefile
@@ -92,7 +92,7 @@ lz4programs: lz4.c lz4hc.c
 liblz4: lz4.c lz4hc.c
 	@echo compiling static library
 	@$(CC) $(CFLAGS) -c $^
-	@ar rcs liblz4.a lz4.o lz4hc.o
+	@$(AR) rcs liblz4.a lz4.o lz4hc.o
 	@echo compiling dynamic library
 	@$(CC) $(CFLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER)
 	@echo creating versioned links
diff --git a/lz4.c b/lz4.c
index f717ca9..00c2a64 100755
--- a/lz4.c
+++ b/lz4.c
@@ -418,7 +418,7 @@ static unsigned LZ4_count(const BYTE* pIn, const BYTE* pRef, const BYTE* pInLimi
 }
 
 
-int LZ4_compress_generic(
+static int LZ4_compress_generic(
                  void* ctx,
                  const char* source,
                  char* dest,
@@ -638,10 +638,9 @@ int LZ4_compress_limitedOutput_withState (void* state, const char* source, char*
    Experimental : Streaming functions
 *****************************************/
 
-void LZ4_renormDictT(LZ4_dict_t_internal* LZ4_dict, const char* source)
+void LZ4_renormDictT(LZ4_dict_t_internal* LZ4_dict)
 {
-    if ((source - LZ4_dict->currentOffset > source)
-        || (LZ4_dict->currentOffset > 0x80000000))
+    if (LZ4_dict->currentOffset > 0x80000000)
     {
         /* rescale hash table */
         U32 delta = LZ4_dict->currentOffset - 64 KB;
@@ -656,24 +655,20 @@ void LZ4_renormDictT(LZ4_dict_t_internal* LZ4_dict, const char* source)
 }
 
 
-static int LZ4_compress_usingDict_generic (LZ4_dict_t* LZ4_dict,
+static inline int LZ4_compress_2_generic (LZ4_dict_t* LZ4_dict,
     const char* source, char* dest, int inputSize, int maxOutputSize,
-    limitedOutput_directive outputLimited)
+    const limitedOutput_directive outputLimited,
+    const dict_directive dict)
 {
     LZ4_dict_t_internal* const streamPtr = (LZ4_dict_t_internal*)LZ4_dict;
     const tableType_t tableType = byU32;
-    U32 currentOffset;
+    const U32 currentOffset = streamPtr->currentOffset;
     const U32 dictSize = streamPtr->dictSize;
     const BYTE* const dictionary = streamPtr->dictionary;
 
-    if (streamPtr->initCheck) return 0;   /* structure not initialized */
-
-    LZ4_renormDictT(streamPtr, source);
-    currentOffset = streamPtr->currentOffset;
-
     streamPtr->dictionary = (const BYTE*)source;
     streamPtr->dictSize = (U32)inputSize;
-    streamPtr->currentOffset += inputSize;
+    streamPtr->currentOffset += (U32)inputSize;
 
     {
         U32 ipIndex = currentOffset;
@@ -721,7 +716,7 @@ static int LZ4_compress_usingDict_generic (LZ4_dict_t* LZ4_dict,
                 if (unlikely(ipIndex > indexLimit)) goto _last_literals;
 
                 ip = base + ipIndex;
-                if (refIndex < currentOffset)
+                if ((dict==usingDict) && (refIndex < currentOffset))
                 {
                     ref = dictBase + refIndex;
                     lowLimit = dictionary;
@@ -729,13 +724,13 @@ static int LZ4_compress_usingDict_generic (LZ4_dict_t* LZ4_dict,
                 else
                 {
                     ref = base + refIndex;
-                    lowLimit = (const BYTE*)source;
+                    if (dict==usingDict) lowLimit = (const BYTE*)source;
                 }
 
             } while ((refIndex + MAX_DISTANCE < ipIndex) || (A32(ref) != A32(ip)));
 
             /* Catch up */
-            while ((ip>anchor) && (ref>lowLimit) && (unlikely(ip[-1]==ref[-1]))) { ip--; ref--; }
+            while ((ip>anchor) && ((dict==usingDict)?(ref>lowLimit):1) && (unlikely(ip[-1]==ref[-1]))) { ip--; ref--; }
 
             {
                 /* Encode Literal length */
@@ -762,9 +757,7 @@ static int LZ4_compress_usingDict_generic (LZ4_dict_t* LZ4_dict,
             /* Encode MatchLength */
             {
                 unsigned matchLength;
-                if (refIndex >= currentOffset)
-                    matchLength = LZ4_count(ip+MINMATCH, ref+MINMATCH, matchlimit);
-                else
+                if ((dict==usingDict) && (refIndex < currentOffset))
                 {
                     const BYTE* dicLimit = ip + (dictEnd - ref);
                     if (dicLimit > matchlimit) dicLimit = matchlimit;
@@ -772,6 +765,10 @@ static int LZ4_compress_usingDict_generic (LZ4_dict_t* LZ4_dict,
                     if (ref + MINMATCH + matchLength == dictEnd)
                         matchLength += LZ4_count(ip+MINMATCH+matchLength, (const BYTE*)source, matchlimit);
                 }
+                else
+                {
+                    matchLength = LZ4_count(ip+MINMATCH, ref+MINMATCH, matchlimit);
+                }
                 ip += matchLength + MINMATCH;
                 if (matchLength>=ML_MASK)
                 {
@@ -811,6 +808,22 @@ static int LZ4_compress_usingDict_generic (LZ4_dict_t* LZ4_dict,
 }
 
 
+static inline int LZ4_compress_usingDict_generic (LZ4_dict_t* LZ4_dict,
+    const char* source, char* dest, int inputSize, int maxOutputSize,
+    const limitedOutput_directive outputLimited)
+{
+    LZ4_dict_t_internal* const streamPtr = (LZ4_dict_t_internal*)LZ4_dict;
+
+    if (streamPtr->initCheck) return 0;   /* structure not initialized */
+    LZ4_renormDictT(streamPtr);
+
+    if ((streamPtr->dictionary + streamPtr->dictSize == (const BYTE*)source)  && (streamPtr->dictSize >= 64 KB))
+    return LZ4_compress_2_generic(LZ4_dict, source, dest, inputSize, maxOutputSize, outputLimited, withPrefix64k);
+    else
+    return LZ4_compress_2_generic(LZ4_dict, source, dest, inputSize, maxOutputSize, outputLimited, usingDict);
+}
+
+
 int LZ4_compress_usingDict (LZ4_dict_t* LZ4_dict, const char* source, char* dest, int inputSize)
 {
     return LZ4_compress_usingDict_generic(LZ4_dict, source, dest, inputSize, 0, notLimited);
@@ -821,6 +834,12 @@ int LZ4_compress_limitedOutput_usingDict (LZ4_dict_t* LZ4_dict, const char* sour
     return LZ4_compress_usingDict_generic(LZ4_dict, source, dest, inputSize, maxOutputSize, limitedOutput);
 }
 
+// Debug function only, to measure performance differences
+int LZ4_compress_forceDict (LZ4_dict_t* LZ4_dict, const char* source, char* dest, int inputSize)
+{
+    return LZ4_compress_2_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, usingDict);
+}
+
 
 int LZ4_setDictPos (LZ4_dict_t* LZ4_dict, const char* dictionary, int dictSize)
 {
@@ -833,7 +852,22 @@ int LZ4_setDictPos (LZ4_dict_t* LZ4_dict, const char* dictionary, int dictSize)
 }
 
 
-int LZ4_loadDict   (LZ4_dict_t* LZ4_dict, const char* dictionary, int dictSize)
+int LZ4_moveDict (LZ4_dict_t* LZ4_dict, char* safeBuffer, int dictSize)
+{
+    LZ4_dict_t_internal* dict = (LZ4_dict_t_internal*) LZ4_dict;
+    const BYTE* previousDictEnd = dict->dictionary + dict->dictSize;
+
+    if ((U32)dictSize > 64 KB) dictSize = 64 KB;   /* useless to define a dictionary > 64 KB */
+    if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize;
+
+    memcpy(safeBuffer, previousDictEnd - dictSize, dictSize);
+    dict->dictSize = (U32)dictSize;
+
+    return 1;
+}
+
+
+int LZ4_loadDict (LZ4_dict_t* LZ4_dict, const char* dictionary, int dictSize)
 {
     LZ4_dict_t_internal* dict = (LZ4_dict_t_internal*) LZ4_dict;
     const BYTE* p = (const BYTE*)dictionary;
diff --git a/programs/fullbench.c b/programs/fullbench.c
index cc39b15..ef42c80 100755
--- a/programs/fullbench.c
+++ b/programs/fullbench.c
@@ -300,6 +300,12 @@ static int local_LZ4_compress_limitedOutput_usingDict(const char* in, char* out,
     return LZ4_compress_limitedOutput_usingDict(&LZ4_dict, in, out, inSize, LZ4_compressBound(inSize));
 }
 
+int LZ4_compress_forceDict (LZ4_dict_t* LZ4_dict, const char* source, char* dest, int inputSize);
+static int local_LZ4_compress_forceDict(const char* in, char* out, int inSize)
+{
+    return LZ4_compress_forceDict(&LZ4_dict, in, out, inSize);
+}
+
 
 static void* stateLZ4HC;
 static int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize)
@@ -364,7 +370,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
 {
   int fileIdx=0;
   char* orig_buff;
-# define NB_COMPRESSION_ALGORITHMS 14
+# define NB_COMPRESSION_ALGORITHMS 15
 # define MINCOMPRESSIONCHAR '0'
 # define MAXCOMPRESSIONCHAR (MINCOMPRESSIONCHAR + NB_COMPRESSION_ALGORITHMS)
   double totalCTime[NB_COMPRESSION_ALGORITHMS+1] = {0};
@@ -504,6 +510,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
             case 12: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break;
             case 13: compressionFunction = local_LZ4_compress_usingDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_usingDict"; break;
             case 14: compressionFunction = local_LZ4_compress_limitedOutput_usingDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_limitedOutput_usingDict"; break;
+            case 15: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break;
             default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1;
             }