First version of Block Streaming API : LZ4_compress_usingDict()

5 files changed, 97 insertions, 59 deletions

diff --git a/programs/Makefile b/programs/Makefile
index c0d6d15..a81a701 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -42,15 +42,6 @@ MANDIR=$(PREFIX)/share/man/man1
 LZ4DIR=..
 
 TEST_FILES = COPYING
-TEST_TARGETS=test-32 test-64
-
-# Minimize test target for Travis CI's Build Matrix
-ifeq ($(LZ4_TRAVIS_CI_ENV),-m32)
-TEST_TARGETS=test-32
-else ifeq ($(LZ4_TRAVIS_CI_ENV),-m64)
-TEST_TARGETS=test-64
-endif
-
 
 # Define *.exe as extension for Windows systems
 ifneq (,$(filter Windows%,$(OS)))
@@ -115,11 +106,7 @@ uninstall:
 	[ -f $(DESTDIR)$(MANDIR)/lz4cat.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4cat.1
 	@echo lz4 successfully uninstalled
 
-test: $(TEST_TARGETS)
-
-test-32: test-lz4 test-lz4c32 test-fullbench32 test-fuzzer32
-
-test-64: test-lz4 test-lz4c test-fullbench test-fuzzer
+test: test-lz4 test-lz4c test-lz4c32 test-fullbench test-fullbench32 test-fuzzer test-fuzzer32
 
 test-lz4:
 
@@ -128,10 +115,10 @@ test-lz4c:
 test-lz4c32:
 
 test-fullbench: fullbench
-	./fullbench --no-prompt $(TEST_FILES)
+	./fullbench --no-prompt -i1 $(TEST_FILES)
 
 test-fullbench32: fullbench32
-	./fullbench32 --no-prompt $(TEST_FILES)
+	./fullbench32 --no-prompt -i1 $(TEST_FILES)
 
 test-fuzzer: fuzzer
 	./fuzzer --no-prompt
diff --git a/programs/fullbench.c b/programs/fullbench.c
index 1200ca0..01b807c 100644
--- a/programs/fullbench.c
+++ b/programs/fullbench.c
@@ -281,6 +281,21 @@ static inline int local_LZ4_compress_limitedOutput_continue(const char* in, char
     return LZ4_compress_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize));
 }
 
+
+LZ4_dict_t LZ4_dict;
+static inline void* local_LZ4_resetDictT(const char* fake)
+{
+    (void)fake;
+    memset(&LZ4_dict, 0, sizeof(LZ4_dict_t));
+    return NULL;
+}
+
+static inline int local_LZ4_compress_usingDict(const char* in, char* out, int inSize)
+{
+    return LZ4_compress_usingDict(&LZ4_dict, in, out, inSize);
+}
+
+
 static void* stateLZ4HC;
 static inline int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize)
 {
@@ -344,15 +359,9 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
 {
   int fileIdx=0;
   char* orig_buff;
-# define NB_COMPRESSION_ALGORITHMS 12
+# define NB_COMPRESSION_ALGORITHMS 13
 # define MINCOMPRESSIONCHAR '0'
 # define MAXCOMPRESSIONCHAR (MINCOMPRESSIONCHAR + NB_COMPRESSION_ALGORITHMS)
-  static char* compressionNames[] = { "LZ4_compress", "LZ4_compress_limitedOutput",
-                                      "LZ4_compress_withState", "LZ4_compress_limitedOutput_withState",
-                                      "LZ4_compress_continue", "LZ4_compress_limitedOutput_continue",
-                                      "LZ4_compressHC", "LZ4_compressHC_limitedOutput",
-                                      "LZ4_compressHC_withStateHC", "LZ4_compressHC_limitedOutput_withStateHC",
-                                      "LZ4_compressHC_continue", "LZ4_compressHC_limitedOutput_continue" };
   double totalCTime[NB_COMPRESSION_ALGORITHMS] = {0};
   double totalCSize[NB_COMPRESSION_ALGORITHMS] = {0};
 # define NB_DECOMPRESSION_ALGORITHMS 7
@@ -465,29 +474,30 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
         DISPLAY(" %s : \n", inFileName);
 
         // Compression Algorithms
-        for (cAlgNb=0; (cAlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); cAlgNb++)
+        for (cAlgNb=1; (cAlgNb <= NB_COMPRESSION_ALGORITHMS) && (compressionTest); cAlgNb++)
         {
-            char* cName = compressionNames[cAlgNb];
+            char* compressorName;
             int (*compressionFunction)(const char*, char*, int);
             void* (*initFunction)(const char*) = NULL;
             double bestTime = 100000000.;
 
-            if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != cAlgNb+1)) continue;
+            if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != cAlgNb)) continue;
 
             switch(cAlgNb)
             {
-            case 0 : compressionFunction = LZ4_compress; break;
-            case 1 : compressionFunction = local_LZ4_compress_limitedOutput; break;
-            case 2 : compressionFunction = local_LZ4_compress_withState; break;
-            case 3 : compressionFunction = local_LZ4_compress_limitedOutput_withState; break;
-            case 4 : compressionFunction = local_LZ4_compress_continue; initFunction = LZ4_create; break;
-            case 5 : compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = LZ4_create; break;
-            case 6 : compressionFunction = LZ4_compressHC; break;
-            case 7 : compressionFunction = local_LZ4_compressHC_limitedOutput; break;
-            case 8 : compressionFunction = local_LZ4_compressHC_withStateHC; break;
-            case 9 : compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; break;
-            case 10: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; break;
-            case 11: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; break;
+            case 1 : compressionFunction = LZ4_compress; compressorName = "LZ4_compress"; break;
+            case 2 : compressionFunction = local_LZ4_compress_limitedOutput; compressorName = "LZ4_compress_limitedOutput"; break;
+            case 3 : compressionFunction = local_LZ4_compress_withState; compressorName = "LZ4_compress_withState"; break;
+            case 4 : compressionFunction = local_LZ4_compress_limitedOutput_withState; compressorName = "LZ4_compress_limitedOutput_withState"; break;
+            case 5 : compressionFunction = local_LZ4_compress_continue; initFunction = LZ4_create; compressorName = "LZ4_compress_continue"; break;
+            case 6 : compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = LZ4_create; compressorName = "LZ4_compress_limitedOutput_continue"; break;
+            case 7 : compressionFunction = LZ4_compressHC; compressorName = "LZ4_compressHC"; break;
+            case 8 : compressionFunction = local_LZ4_compressHC_limitedOutput; compressorName = "LZ4_compressHC_limitedOutput"; break;
+            case 9 : compressionFunction = local_LZ4_compressHC_withStateHC; compressorName = "LZ4_compressHC_withStateHC"; break;
+            case 10: compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; compressorName = "LZ4_compressHC_limitedOutput_withStateHC"; break;
+            case 11: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_continue"; break;
+            case 12: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break;
+            case 13: compressionFunction = local_LZ4_compress_usingDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_usingDict"; break;
             default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1;
             }
 
@@ -496,7 +506,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
                 double averageTime;
                 int milliTime;
 
-                PROGRESS("%1i-%-21.21s : %9i ->\r", loopNb, cName, (int)benchedSize);
+                PROGRESS("%1i-%-25.25s : %9i ->\r", loopNb, compressorName, (int)benchedSize);
                 { size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; }     // warmimg up memory
 
                 nb_loops = 0;
@@ -509,7 +519,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
                     for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
                     {
                         chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
-                        if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", cName), exit(1);
+                        if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", compressorName), exit(1);
                     }
                     if (initFunction!=NULL) free(ctx);
                     nb_loops++;
@@ -520,13 +530,13 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
                 if (averageTime < bestTime) bestTime = averageTime;
                 cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;
                 ratio = (double)cSize/(double)benchedSize*100.;
-                PROGRESS("%1i-%-21.21s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
+                PROGRESS("%1i-%-25.25s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
             }
 
             if (ratio<100.)
-                DISPLAY("%-23.23s : %9i -> %9i (%5.2f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
+                DISPLAY("%-27.27s : %9i -> %9i (%5.2f%%),%7.1f MB/s\n", compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
             else
-                DISPLAY("%-23.23s : %9i -> %9i (%5.1f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
+                DISPLAY("%-27.27s : %9i -> %9i (%5.1f%%),%7.1f MB/s\n", compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
 
             totalCTime[cAlgNb] += bestTime;
             totalCSize[cAlgNb] += cSize;
@@ -536,7 +546,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
         for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
         {
             chunkP[chunkNb].compressedSize = LZ4_compress(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
-            if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", compressionNames[0]), exit(1);
+            if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress"), exit(1);
         }
         { size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; }     // zeroing source area, for CRC checking
 
@@ -607,6 +617,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
       free(chunkP);
   }
 
+/*
   if (nbFiles > 1)
   {
       int AlgNb;
@@ -625,6 +636,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
           DISPLAY("%-31.31s :%10llu -> %6.1f MB/s\n", dName, (long long unsigned int)totals, (double)totals/totalDTime[AlgNb]/1000.);
       }
   }
+*/
 
   if (BMK_pause) { printf("press enter...\n"); getchar(); }
 
diff --git a/programs/fuzzer.c b/programs/fuzzer.c
index 1ea14f6..081f0df 100644
--- a/programs/fuzzer.c
+++ b/programs/fuzzer.c
@@ -187,7 +187,7 @@ int FUZ_SecurityTest()
 
 #define FUZ_MAX(a,b) (a>b?a:b)
 
-int FUZ_test(U32 seed, int nbTests, double compressibility) {
+int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) {
         unsigned long long bytes = 0;
         unsigned long long cbytes = 0;
         unsigned long long hcbytes = 0;
@@ -197,10 +197,10 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) {
         char* decodedBuffer;
 #       define FUZ_max   LZ4_COMPRESSBOUND(LEN)
         unsigned int randState=seed;
-        int ret, attemptNb;
+        int ret, cycleNb;
 #       define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %i : ", testNb); printf(__VA_ARGS__); \
-                                        printf(" (seed %u, cycle %i) \n", seed, attemptNb); goto _output_error; }
-#       define FUZ_DISPLAYTEST          testNb++; no_prompt ? 0 : printf("%2i\b\b", testNb);
+                                        printf(" (seed %u, cycle %i) \n", seed, cycleNb); goto _output_error; }
+#       define FUZ_DISPLAYTEST          { testNb++; no_prompt ? 0 : printf("%2i\b\b", testNb); }
         void* stateLZ4   = malloc(LZ4_sizeofState());
         void* stateLZ4HC = malloc(LZ4_sizeofStateHC());
         void* LZ4continue;
@@ -213,8 +213,16 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) {
         compressedBuffer = malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE));
         decodedBuffer = malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE);
 
+        // move to startCycle
+        for (cycleNb = 0; cycleNb < startCycle; cycleNb++)
+        {
+            FUZ_rand(&randState);
+            FUZ_rand(&randState);
+            FUZ_rand(&randState);
+        }
+
         // Test loop
-        for (attemptNb = 0; attemptNb < nbTests; attemptNb++)
+        for (cycleNb = startCycle; cycleNb < nbCycles; cycleNb++)
         {
             int testNb = 0;
             char* dict;
@@ -224,11 +232,11 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) {
 
             // note : promptThrottle is throtting stdout to prevent
             //        Travis-CI's output limit (10MB) and false hangup detection.
-            const int step = FUZ_MAX(1, nbTests / 100);
-            const int promptThrottle = ((attemptNb % step) == 0);
-            if (!no_prompt || attemptNb == 0 || promptThrottle)
+            const int step = FUZ_MAX(1, nbCycles / 100);
+            const int promptThrottle = ((cycleNb % step) == 0);
+            if (!no_prompt || cycleNb == 0 || promptThrottle)
             {
-                printf("\r%7i /%7i   - ", attemptNb, nbTests);
+                printf("\r%7i /%7i   - ", cycleNb, nbCycles);
                 if (no_prompt) fflush(stdout);
             }
 
@@ -305,7 +313,7 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) {
             ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize+1);
             FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite amply sufficient space");
             FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data");
-            //FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe wrote more than target size");   // well, is that an issue ?
+            //FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe wrote more than (unknown) target size");   // well, is that an issue ?
             FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ4_decompress_safe overrun specified output buffer size");
             crcCheck = XXH32(decodedBuffer, blockSize, 0);
             FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
@@ -409,6 +417,18 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) {
             crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
             FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_withPrefix64k corrupted decoded data");
 
+            // Compress using dictionary
+            FUZ_DISPLAYTEST;
+            dict -= 9;
+            if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
+            {
+                LZ4_dict_t LZ4dict;
+                memset(&LZ4dict, 0, sizeof(LZ4_dict_t));
+                LZ4_loadDict(&LZ4dict, dict, dictSize);
+                blockContinueCompressedSize = LZ4_compress_usingDict(&LZ4dict, block, compressedBuffer, blockSize);
+                FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_usingDict failed");
+            }
+
             // Decompress with dictionary as external
             FUZ_DISPLAYTEST;
             decodedBuffer[blockSize] = 0;
@@ -416,6 +436,13 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) {
             FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
             FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size")
             crcCheck = XXH32(decodedBuffer, blockSize, 0);
+            if (crcCheck!=crcOrig)
+            {
+                int i=0;
+                while (block[i]==decodedBuffer[i]) i++;
+                printf("Wrong Byte at position %i/%i\n", i, blockSize);
+
+            }
             FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data");
 
             FUZ_DISPLAYTEST;
@@ -455,7 +482,7 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) {
             ccbytes += blockContinueCompressedSize;
         }
 
-        printf("\r%7i /%7i   - ", attemptNb, nbTests);
+        printf("\r%7i /%7i   - ", cycleNb, nbCycles);
         printf("all tests completed successfully \n");
         printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100);
         printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100);
@@ -489,6 +516,7 @@ int FUZ_usage()
     DISPLAY( "Arguments :\n");
     DISPLAY( " -i#    : Nb of tests (default:%i) \n", NB_ATTEMPTS);
     DISPLAY( " -s#    : Select seed (default:prompt user)\n");
+    DISPLAY( " -t#    : Select starting test number (default:0)\n");
     DISPLAY( " -p#    : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
     DISPLAY( " -h     : display help and exit\n");
     return 0;
@@ -502,6 +530,7 @@ int main(int argc, char** argv) {
     int seedset=0;
     int argNb;
     int nbTests = NB_ATTEMPTS;
+    int testNb = 0;
     int proba = FUZ_COMPRESSIBILITY_DEFAULT;
 
     // Check command line
@@ -544,6 +573,16 @@ int main(int argc, char** argv) {
                         argument++;
                     }
                     break;
+                case 't':
+                    argument++;
+                    testNb=0;
+                    while ((*argument>='0') && (*argument<='9'))
+                    {
+                        testNb *= 10;
+                        testNb += *argument - '0';
+                        argument++;
+                    }
+                    break;
                 case 'p':
                     argument++;
                     proba=0;
@@ -583,5 +622,5 @@ int main(int argc, char** argv) {
 
     if (nbTests<=0) nbTests=1;
 
-    return FUZ_test(seed, nbTests, ((double)proba) / 100);
+    return FUZ_test(seed, nbTests, testNb, ((double)proba) / 100);
 }
diff --git a/programs/lz4.1 b/programs/lz4.1
index 298cbf6..6ae8d3c 100644
--- a/programs/lz4.1
+++ b/programs/lz4.1
@@ -64,6 +64,7 @@ following options
 .TP
 .B \-B#
  block size [4-7](default : 7)
+ B4= 64KB ; B5= 256KB ; B6= 1MB ; B7= 4MB
 .TP
 .B \-BD
  block dependency (improve compression ratio)
@@ -84,4 +85,4 @@ following options
 Report bugs at:- https://code.google.com/p/lz4/
 
 .SH AUTHOR
-Yann Collet
\ No newline at end of file
+Yann Collet
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index 934c2bb..1c4e9de 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -405,9 +405,8 @@ int main(int argc, char** argv)
                         case '7':
                         {
                             int B = argument[1] - '0';
-                            int S = 1 << (8 + 2*B);
-                            BMK_SetBlocksize(S);
                             blockSize = LZ4IO_setBlockSizeID(B);
+                            BMK_SetBlocksize(blockSize);
                             argument++;
                             break;
                         }