converge towards LZ4_compress_continue()

4 files changed, 39 insertions, 114 deletions

diff --git a/lz4.c b/lz4.c
index 2b37c69..6eff9b9 100755
--- a/lz4.c
+++ b/lz4.c
@@ -751,59 +751,6 @@ void LZ4_renormDictT(LZ4_dict_t_internal* LZ4_dict, const BYTE* src)
 }
 
 
-int LZ4_compress_usingDict (void* LZ4_dict, const char* source, char* dest, int inputSize)
-{
-    LZ4_dict_t_internal* streamPtr = (LZ4_dict_t_internal*)LZ4_dict;
-    const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
-
-    const BYTE* smallest = dictEnd;
-    if (smallest > (const BYTE*) source) smallest = (const BYTE*) source;
-    LZ4_renormDictT((LZ4_dict_t_internal*)LZ4_dict, smallest);
-
-    if (dictEnd == (const BYTE*)source)
-    {
-        int result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, byU32, withPrefix64k);
-        streamPtr->dictSize += (U32)inputSize;
-        streamPtr->currentOffset += (U32)inputSize;
-        return result;
-    }
-
-    {
-        int result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, byU32, usingExtDict);
-        streamPtr->dictionary = (const BYTE*)source;
-        streamPtr->dictSize = (U32)inputSize;
-        streamPtr->currentOffset += (U32)inputSize;
-        return result;
-    }
-}
-
-int LZ4_compress_limitedOutput_usingDict (void* LZ4_dict, const char* source, char* dest, int inputSize, int maxOutputSize)
-{
-    LZ4_dict_t_internal* streamPtr = (LZ4_dict_t_internal*)LZ4_dict;
-    const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
-
-    const BYTE* smallest = dictEnd;
-    if (smallest > (const BYTE*) source) smallest = (const BYTE*) source;
-    LZ4_renormDictT((LZ4_dict_t_internal*)LZ4_dict, smallest);
-
-    if (dictEnd == (const BYTE*)source)
-    {
-        int result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k);
-        streamPtr->dictSize += (U32)inputSize;
-        streamPtr->currentOffset += (U32)inputSize;
-        return result;
-    }
-
-    {
-        int result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict);
-        streamPtr->dictionary = (const BYTE*)source;
-        streamPtr->dictSize = (U32)inputSize;
-        streamPtr->currentOffset += (U32)inputSize;
-        return result;
-    }
-}
-
-
 int LZ4_compress_continue (void* LZ4_stream, const char* source, char* dest, int inputSize)
 {
     LZ4_dict_t_internal* streamPtr = (LZ4_dict_t_internal*)LZ4_stream;
diff --git a/lz4.h b/lz4.h
index 7e38a54..5afc133 100644
--- a/lz4.h
+++ b/lz4.h
@@ -183,6 +183,7 @@ typedef struct { unsigned int table[LZ4_DICTSIZE_U32]; } LZ4_dict_t;
 /*
  * LZ4_createStream
  * provides a pointer (void*) towards an initialized LZ4_dict_t structure
+ * LZ4_free just frees it.
  */
 void* LZ4_createStream();
 int   LZ4_free (void* LZ4_stream);
@@ -190,30 +191,30 @@ int   LZ4_free (void* LZ4_stream);
 /*
  * LZ4_loadDict
  * Use this function to load a static dictionary into LZ4_dict.
- * You can load a size of 0 to init an LZ4_dict_t structure
+ * Loading a size of 0 is allowed and init the LZ4_dict_t structure.
  * Return : 1 if OK, 0 if error
  */
 int LZ4_loadDict (void* LZ4_stream, const char* dictionary, int dictSize);
 
 /*
- * LZ4_compress_usingDict
+ * LZ4_compress_continue
  * Compress data block 'source', using blocks compressed before to improve compression ratio
  * Previous data blocks are assumed to still be present at their previous location.
  */
-int LZ4_compress_usingDict (void* LZ4_stream, const char* source, char* dest, int inputSize);
+int LZ4_compress_continue (void* LZ4_stream, const char* source, char* dest, int inputSize);
 
 /*
- * LZ4_compress_limitedOutput_usingDict
+ * LZ4_compress_limitedOutput_continue
  * Same as before, but also specify a maximum target compressed size (maxOutputSize)
  * If it cannot be met, compression exits, and return a zero.
  */
-int LZ4_compress_limitedOutput_usingDict (void* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize);
+int LZ4_compress_limitedOutput_continue (void* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize);
 
 /*
  * LZ4_moveDict
  * If previously compressed data block is not guaranteed to remain at its previous memory location
  * save it into a safe place (char* safeBuffer)
- * before calling again LZ4_compress_usingDict()
+ * before calling again LZ4_compress_continue()
  * Return : 1 if OK, 0 if error
  * Note : any dictSize > 64 KB will be interpreted as 64KB.
  */
@@ -240,14 +241,15 @@ int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int compr
 int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int originalSize);
 
 
+
 /**************************************
    Obsolete Functions
 **************************************/
 /*
 These function names are deprecated and should no longer be used.
 They are only provided here for compatibility with older user programs.
-- LZ4_uncompress is totally equivalent to LZ4_decompress_fast
-- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe
+- LZ4_uncompress is the same as LZ4_decompress_fast
+- LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe
 */
 int   LZ4_uncompress (const char* source, char* dest, int outputSize);
 int   LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
@@ -256,9 +258,7 @@ int   LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isiz
 void* LZ4_create (const char* inputBuffer);
 int   LZ4_sizeofStreamState(void);
 int   LZ4_resetStreamState(void* state, const char* inputBuffer);
-int   LZ4_compress_continue (void* LZ4_Data, const char* source, char* dest, int inputSize);
-int   LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
-char* LZ4_slideInputBuffer (void* LZ4_Data);
+char* LZ4_slideInputBuffer (void* state);
 
 
 #if defined (__cplusplus)
diff --git a/programs/fullbench.c b/programs/fullbench.c
index 7249387..23ca5b7 100755
--- a/programs/fullbench.c
+++ b/programs/fullbench.c
@@ -290,16 +290,6 @@ static void* local_LZ4_resetDictT(const char* fake)
     return NULL;
 }
 
-static int local_LZ4_compress_usingDict(const char* in, char* out, int inSize)
-{
-    return LZ4_compress_usingDict(&LZ4_dict, in, out, inSize);
-}
-
-static int local_LZ4_compress_limitedOutput_usingDict(const char* in, char* out, int inSize)
-{
-    return LZ4_compress_limitedOutput_usingDict(&LZ4_dict, in, out, inSize, LZ4_compressBound(inSize));
-}
-
 int LZ4_compress_forceExtDict (LZ4_dict_t* LZ4_dict, const char* source, char* dest, int inputSize);
 static int local_LZ4_compress_forceDict(const char* in, char* out, int inSize)
 {
@@ -370,9 +360,8 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
 {
   int fileIdx=0;
   char* orig_buff;
-# define NB_COMPRESSION_ALGORITHMS 15
+# define NB_COMPRESSION_ALGORITHMS 13
 # define MINCOMPRESSIONCHAR '0'
-# define MAXCOMPRESSIONCHAR (MINCOMPRESSIONCHAR + NB_COMPRESSION_ALGORITHMS)
   double totalCTime[NB_COMPRESSION_ALGORITHMS+1] = {0};
   double totalCSize[NB_COMPRESSION_ALGORITHMS+1] = {0};
 # define NB_DECOMPRESSION_ALGORITHMS 7
@@ -508,9 +497,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
             case 10: compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; compressorName = "LZ4_compressHC_limitedOutput_withStateHC"; break;
             case 11: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_continue"; break;
             case 12: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break;
-            case 13: compressionFunction = local_LZ4_compress_usingDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_usingDict"; break;
-            case 14: compressionFunction = local_LZ4_compress_limitedOutput_usingDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_limitedOutput_usingDict"; break;
-            case 15: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break;
+            case 13: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break;
             default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1;
             }
 
@@ -630,27 +617,6 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
       free(chunkP);
   }
 
-/*
-  if (nbFiles > 1)
-  {
-      int AlgNb;
-
-      DISPLAY(" ** TOTAL ** : \n");
-      for (AlgNb = 0; (AlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); AlgNb ++)
-      {
-          char* cName = compressionNames[AlgNb];
-          if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != AlgNb)) continue;
-          DISPLAY("%-23.23s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s\n", cName, (long long unsigned int)totals, (long long unsigned int)totalCSize[AlgNb], (double)totalCSize[AlgNb]/(double)totals*100., (double)totals/totalCTime[AlgNb]/1000.);
-      }
-      for (AlgNb = 0; (AlgNb < NB_DECOMPRESSION_ALGORITHMS) && (decompressionTest); AlgNb ++)
-      {
-          char* dName = decompressionNames[AlgNb];
-          if ((decompressionAlgo != ALL_DECOMPRESSORS) && (decompressionAlgo != AlgNb)) continue;
-          DISPLAY("%-31.31s :%10llu -> %6.1f MB/s\n", dName, (long long unsigned int)totals, (double)totals/totalDTime[AlgNb]/1000.);
-      }
-  }
-*/
-
   if (BMK_pause) { printf("press enter...\n"); getchar(); }
 
   return 0;
diff --git a/programs/fuzzer.c b/programs/fuzzer.c
index 56208d3..608340c 100644
--- a/programs/fuzzer.c
+++ b/programs/fuzzer.c
@@ -83,11 +83,12 @@
 #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
 
 
-//**************************************
-// Local Parameters
-//**************************************
+/*****************************************
+  Local Parameters
+*****************************************/
 static int no_prompt = 0;
 static char* programName;
+static int displayLevel = 2;
 
 
 /*********************************************************
@@ -200,12 +201,13 @@ int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) {
         int ret, cycleNb;
 #       define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %i : ", testNb); printf(__VA_ARGS__); \
                                         printf(" (seed %u, cycle %i) \n", seed, cycleNb); goto _output_error; }
-#       define FUZ_DISPLAYTEST          { testNb++; no_prompt ? 0 : printf("%2i\b\b", testNb); }
+#       define FUZ_DISPLAYTEST          { testNb++; ((displayLevel<3) || no_prompt) ? 0 : printf("%2i\b\b", testNb); if (displayLevel==4) fflush(stdout); }
         void* stateLZ4   = malloc(LZ4_sizeofState());
         void* stateLZ4HC = malloc(LZ4_sizeofStateHC());
         void* LZ4continue;
         LZ4_dict_t LZ4dict;
         U32 crcOrig, crcCheck;
+        int displayRefresh;
 
 
         // Create compressible test buffer
@@ -214,6 +216,15 @@ int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) {
         compressedBuffer = malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE));
         decodedBuffer = malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE);
 
+        // display refresh rate
+        switch(displayLevel)
+        {
+        case 0: displayRefresh = nbCycles+1; break;
+        case 1: displayRefresh=FUZ_MAX(1, nbCycles / 100); break;
+        case 2: displayRefresh=99; break;
+        default : displayRefresh=1;
+        }
+
         // move to startCycle
         for (cycleNb = 0; cycleNb < startCycle; cycleNb++)
         {
@@ -231,14 +242,10 @@ int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) {
             int dictSize, blockSize, blockStart, compressedSize, HCcompressedSize;
             int blockContinueCompressedSize;
 
-            // note : promptThrottle is throtting stdout to prevent
-            //        Travis-CI's output limit (10MB) and false hangup detection.
-            const int step = FUZ_MAX(1, nbCycles / 100);
-            const int promptThrottle = ((cycleNb % step) == 0);
-            if (!no_prompt || cycleNb == 0 || promptThrottle)
+            if ((cycleNb%displayRefresh) == 0)
             {
                 printf("\r%7i /%7i   - ", cycleNb, nbCycles);
-                if (no_prompt) fflush(stdout);
+                fflush(stdout);
             }
 
             // Select block to test
@@ -431,19 +438,19 @@ int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) {
             if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
             memset(&LZ4dict, 0, sizeof(LZ4_dict_t));
             LZ4_loadDict(&LZ4dict, dict, dictSize);
-            blockContinueCompressedSize = LZ4_compress_usingDict(&LZ4dict, block, compressedBuffer, blockSize);
+            blockContinueCompressedSize = LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
             FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_usingDict failed");
 
             FUZ_DISPLAYTEST;
             memset(&LZ4dict, 0, sizeof(LZ4_dict_t));
             LZ4_loadDict(&LZ4dict, dict, dictSize);
-            ret = LZ4_compress_limitedOutput_usingDict(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
+            ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
             FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_usingDict should fail : one missing byte for output buffer");
 
             FUZ_DISPLAYTEST;
             memset(&LZ4dict, 0, sizeof(LZ4_dict_t));
             LZ4_loadDict(&LZ4dict, dict, dictSize);
-            ret = LZ4_compress_limitedOutput_usingDict(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize);
+            ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize);
             FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_usingDict should work : enough size available within output buffer");
 
             // Decompress with dictionary as external
@@ -535,6 +542,7 @@ int FUZ_usage()
     DISPLAY( " -s#    : Select seed (default:prompt user)\n");
     DISPLAY( " -t#    : Select starting test number (default:0)\n");
     DISPLAY( " -p#    : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
+    DISPLAY( " -v     : verbose\n");
     DISPLAY( " -h     : display help and exit\n");
     return 0;
 }
@@ -561,7 +569,7 @@ int main(int argc, char** argv) {
         // Decode command (note : aggregated commands are allowed)
         if (argument[0]=='-')
         {
-            if (!strcmp(argument, "--no-prompt")) { no_prompt=1; seedset=1; continue; }
+            if (!strcmp(argument, "--no-prompt")) { no_prompt=1; seedset=1; displayLevel=1; continue; }
 
             while (argument[1]!=0)
             {
@@ -570,6 +578,10 @@ int main(int argc, char** argv) {
                 {
                 case 'h':
                     return FUZ_usage();
+                case 'v':
+                    argument++;
+                    displayLevel=4;
+                    break;
                 case 'i':
                     argument++;
                     nbTests=0;