fixed lz4cat

changed lz4-test order

6 files changed, 88 insertions, 65 deletions

diff --git a/NEWS b/NEWS
index 8a74594..3257fd5 100644
--- a/NEWS
+++ b/NEWS
@@ -8,13 +8,13 @@ Fix : better ratio on 64-bits big-endian targets
 Improved cmake build script, by Evan Nemerson
 New liblz4-dll project, by Przemyslaw Skibinki
 Makefile: Generates object files (*.o) for faster (re)compilation on low power systems
-Added : Debianization, by Evgeniy Polyakov
 cli : new : --rm command
 cli : new : file attributes are preserved, by Przemyslaw Skibinki
 cli : fix : crash on some invalid inputs
 cli : fix : -t correctly validates lz4-compressed files, by Nick Terrell
 cli : fix : detects and reports fread() errors, thanks to Hiroshi Fujishima report #243
 cli : bench : new : -r recursive mode
+Added : Debianization, by Evgeniy Polyakov
 
 r131
 New    : Dos/DJGPP target, thanks to Louis Santillan (#114)
diff --git a/programs/lz4.1 b/programs/lz4.1
index 529d36a..5a28a83 100644
--- a/programs/lz4.1
+++ b/programs/lz4.1
@@ -22,7 +22,7 @@ is equivalent to
 .br
 .B lz4cat
 is equivalent to
-.BR "lz4 \-dc"
+.BR "lz4 \-dcf"
 .br
 .PP
 When writing scripts that need to decompress files,
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index def7894..fdfa8f2 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -99,12 +99,6 @@ static unsigned displayLevel = 2;   /* 0 : no display ; 1: errors only ; 2 : dow
 
 
 /*-************************************
-*  Local Variables
-***************************************/
-static const char* programName;
-
-
-/*-************************************
 *  Exceptions
 ***************************************/
 #define DEBUG 0
@@ -133,10 +127,10 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
 /*-***************************
 *  Functions
 *****************************/
-static int usage(void)
+static int usage(const char* exeName)
 {
     DISPLAY( "Usage :\n");
-    DISPLAY( "      %s [arg] [input] [output]\n", programName);
+    DISPLAY( "      %s [arg] [input] [output]\n", exeName);
     DISPLAY( "\n");
     DISPLAY( "input   : a filename\n");
     DISPLAY( "          with no FILE, or when FILE is - or %s, read standard input\n", stdinmark);
@@ -151,10 +145,10 @@ static int usage(void)
     return 0;
 }
 
-static int usage_advanced(void)
+static int usage_advanced(const char* exeName)
 {
     DISPLAY(WELCOME_MESSAGE);
-    usage();
+    usage(exeName);
     DISPLAY( "\n");
     DISPLAY( "Advanced arguments :\n");
     DISPLAY( " -V     : display Version number and exit\n");
@@ -188,9 +182,9 @@ static int usage_advanced(void)
     return 0;
 }
 
-static int usage_longhelp(void)
+static int usage_longhelp(const char* exeName)
 {
-    usage_advanced();
+    usage_advanced(exeName);
     DISPLAY( "\n");
     DISPLAY( "****************************\n");
     DISPLAY( "***** Advanced comment *****\n");
@@ -216,43 +210,43 @@ static int usage_longhelp(void)
     DISPLAY( "stdin, stdout and the console : \n");
     DISPLAY( "--------------------------------\n");
     DISPLAY( "To protect the console from binary flooding (bad argument mistake)\n");
-    DISPLAY( "%s will refuse to read from console, or write to console \n", programName);
+    DISPLAY( "%s will refuse to read from console, or write to console \n", exeName);
     DISPLAY( "except if '-c' command is specified, to force output to console \n");
     DISPLAY( "\n");
     DISPLAY( "Simple example :\n");
     DISPLAY( "----------------\n");
     DISPLAY( "1 : compress 'filename' fast, using default output name 'filename.lz4'\n");
-    DISPLAY( "          %s filename\n", programName);
+    DISPLAY( "          %s filename\n", exeName);
     DISPLAY( "\n");
     DISPLAY( "Short arguments can be aggregated. For example :\n");
     DISPLAY( "----------------------------------\n");
     DISPLAY( "2 : compress 'filename' in high compression mode, overwrite output if exists\n");
-    DISPLAY( "          %s -9 -f filename \n", programName);
+    DISPLAY( "          %s -9 -f filename \n", exeName);
     DISPLAY( "    is equivalent to :\n");
-    DISPLAY( "          %s -9f filename \n", programName);
+    DISPLAY( "          %s -9f filename \n", exeName);
     DISPLAY( "\n");
-    DISPLAY( "%s can be used in 'pure pipe mode'. For example :\n", programName);
+    DISPLAY( "%s can be used in 'pure pipe mode'. For example :\n", exeName);
     DISPLAY( "-------------------------------------\n");
     DISPLAY( "3 : compress data stream from 'generator', send result to 'consumer'\n");
-    DISPLAY( "          generator | %s | consumer \n", programName);
+    DISPLAY( "          generator | %s | consumer \n", exeName);
 #if defined(ENABLE_LZ4C_LEGACY_OPTIONS)
     DISPLAY( "\n");
     DISPLAY( "***** Warning  *****\n");
     DISPLAY( "Legacy arguments take precedence. Therefore : \n");
     DISPLAY( "---------------------------------\n");
-    DISPLAY( "          %s -hc filename\n", programName);
+    DISPLAY( "          %s -hc filename\n", exeName);
     DISPLAY( "means 'compress filename in high compression mode'\n");
     DISPLAY( "It is not equivalent to :\n");
-    DISPLAY( "          %s -h -c filename\n", programName);
+    DISPLAY( "          %s -h -c filename\n", exeName);
     DISPLAY( "which would display help text and exit\n");
 #endif /* ENABLE_LZ4C_LEGACY_OPTIONS */
     return 0;
 }
 
-static int badusage(void)
+static int badusage(const char* exeName)
 {
     DISPLAYLEVEL(1, "Incorrect parameters\n");
-    if (displayLevel >= 1) usage();
+    if (displayLevel >= 1) usage(exeName);
     exit(1);
 }
 
@@ -276,28 +270,29 @@ static unsigned readU32FromChar(const char** stringPtr)
     return result;
 }
 
+typedef enum { om_auto, om_compress, om_decompress, om_test, om_bench } operationMode_e;
 
 int main(int argc, const char** argv)
 {
     int i,
         cLevel=1,
         cLevelLast=1,
-        decode=0,
-        bench=0,
         legacy_format=0,
         forceStdout=0,
         forceCompress=0,
         main_pause=0,
         multiple_inputs=0,
         operationResult=0;
+    operationMode_e mode = om_auto;
     const char* input_filename=0;
     const char* output_filename=0;
     char* dynNameSpace=0;
     const char** inFileNames = NULL;
     unsigned ifnIdx=0;
-    char nullOutput[] = NULL_OUTPUT;
-    char extension[] = LZ4_EXTENSION;
-    int  blockSize;
+    const char nullOutput[] = NULL_OUTPUT;
+    const char extension[] = LZ4_EXTENSION;
+    int  blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT);
+    const char* const exeName = argv[0];
 #ifdef UTIL_HAS_CREATEFILELIST
     const char** extendedFileList = NULL;
     char* fileNamesBuf = NULL;
@@ -305,13 +300,11 @@ int main(int argc, const char** argv)
 #endif
 
     /* Init */
-    programName = argv[0];
     LZ4IO_setOverwrite(0);
-    blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT);
 
     /* lz4cat predefined behavior */
-    if (!strcmp(programName, LZ4CAT)) { decode=1; forceStdout=1; output_filename=stdoutmark; displayLevel=1; }
-    if (!strcmp(programName, UNLZ4)) { decode=1; }
+    if (!strcmp(exeName, LZ4CAT)) { mode = om_decompress; LZ4IO_setOverwrite(1); forceStdout=1; output_filename=stdoutmark; displayLevel=1; }
+    if (!strcmp(exeName, UNLZ4)) { mode = om_decompress; }
 
     /* command switches */
     for(i=1; i<argc; i++) {
@@ -322,9 +315,9 @@ int main(int argc, const char** argv)
         /* long commands (--long-word) */
         if (!strcmp(argument,  "--compress")) { forceCompress = 1; continue; }
         if ((!strcmp(argument, "--decompress"))
-         || (!strcmp(argument, "--uncompress"))) { decode = 1; continue; }
+         || (!strcmp(argument, "--uncompress"))) { mode = om_decompress; continue; }
         if (!strcmp(argument,  "--multiple")) { multiple_inputs = 1; if (inFileNames==NULL) inFileNames = (const char**)malloc(argc * sizeof(char*)); continue; }
-        if (!strcmp(argument,  "--test")) { decode = 1; LZ4IO_setTestMode(1); output_filename=nulmark; continue; }
+        if (!strcmp(argument,  "--test")) { mode = om_test; continue; }
         if (!strcmp(argument,  "--force")) { LZ4IO_setOverwrite(1); continue; }
         if (!strcmp(argument,  "--no-force")) { LZ4IO_setOverwrite(0); continue; }
         if ((!strcmp(argument, "--stdout"))
@@ -335,7 +328,7 @@ int main(int argc, const char** argv)
         if (!strcmp(argument,  "--no-content-size")) { LZ4IO_setContentSize(0); continue; }
         if (!strcmp(argument,  "--sparse")) { LZ4IO_setSparseFile(2); continue; }
         if (!strcmp(argument,  "--no-sparse")) { LZ4IO_setSparseFile(0); continue; }
-        if (!strcmp(argument,  "--verbose")) { displayLevel=4; continue; }
+        if (!strcmp(argument,  "--verbose")) { displayLevel++; continue; }
         if (!strcmp(argument,  "--quiet")) { if (displayLevel) displayLevel--; continue; }
         if (!strcmp(argument,  "--version")) { DISPLAY(WELCOME_MESSAGE); return 0; }
         if (!strcmp(argument,  "--keep")) { LZ4IO_setRemoveSrcFile(0); continue; }   /* keep source file (default anyway; just for xz/lzma compatibility) */
@@ -371,8 +364,8 @@ int main(int argc, const char** argv)
                 {
                     /* Display help */
                 case 'V': DISPLAY(WELCOME_MESSAGE); goto _cleanup;   /* Version */
-                case 'h': usage_advanced(); goto _cleanup;
-                case 'H': usage_longhelp(); goto _cleanup;
+                case 'h': usage_advanced(exeName); goto _cleanup;
+                case 'H': usage_longhelp(exeName); goto _cleanup;
 
                 case 'e':
                     argument++;
@@ -381,25 +374,25 @@ int main(int argc, const char** argv)
                     break;
 
                     /* Compression (default) */
-                case 'z': forceCompress = 1; break;
+                case 'z': mode = om_compress; break;
 
                     /* Use Legacy format (ex : Linux kernel compression) */
                 case 'l': legacy_format = 1; blockSize = 8 MB; break;
 
                     /* Decoding */
-                case 'd': decode=1; break;
+                case 'd': mode = om_decompress; break;
 
                     /* Force stdout, even if stdout==console */
                 case 'c': forceStdout=1; output_filename=stdoutmark; displayLevel=1; break;
 
                     /* Test integrity */
-                case 't': decode=1; LZ4IO_setTestMode(1); output_filename=nulmark; break;
+                case 't': mode = om_test; break;
 
                     /* Overwrite */
                 case 'f': LZ4IO_setOverwrite(1); break;
 
                     /* Verbose mode */
-                case 'v': displayLevel=4; break;
+                case 'v': displayLevel++; break;
 
                     /* Quiet mode */
                 case 'q': if (displayLevel) displayLevel--; break;
@@ -433,7 +426,7 @@ int main(int argc, const char** argv)
                     break;
 
                     /* Benchmark */
-                case 'b': bench=1; multiple_inputs=1;
+                case 'b': mode = om_bench; multiple_inputs=1;
                     if (inFileNames == NULL)
                         inFileNames = (const char**) malloc(argc * sizeof(char*));
                     break;
@@ -466,7 +459,7 @@ int main(int argc, const char** argv)
                 EXTENDED_ARGUMENTS;
 
                     /* Unrecognised command */
-                default : badusage();
+                default : badusage(exeName);
                 }
             }
             continue;
@@ -490,7 +483,7 @@ int main(int argc, const char** argv)
     }
 
     DISPLAYLEVEL(3, WELCOME_MESSAGE);
-    if (!decode) DISPLAYLEVEL(4, "Blocks size : %i KB\n", blockSize>>10);
+    if (mode == om_compress) DISPLAYLEVEL(4, "Blocks size : %i KB\n", blockSize>>10);
 
     /* No input filename ==> use stdin */
     if (multiple_inputs) {
@@ -517,42 +510,53 @@ int main(int argc, const char** argv)
         exit(1);
     }
 
-    /* Check if benchmark is selected */
-    if (bench) {
+    /* benchmark and test modes */
+    if (mode == om_bench) {
         BMK_setNotificationLevel(displayLevel);
         operationResult = BMK_benchFiles(inFileNames, ifnIdx, cLevel, cLevelLast);
         goto _cleanup;
     }
 
+    if (mode == om_test) {
+        LZ4IO_setTestMode(1);
+        output_filename=nulmark;
+        mode = om_decompress;   /* defer to decompress */
+    }
+
+    /* compress or decompress */
     /* No output filename ==> try to select one automatically (when possible) */
     while (!output_filename) {
         if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; break; }   /* Default to stdout whenever possible (i.e. not a console) */
-        if ((!decode) && !(forceCompress)) {  /* auto-determine compression or decompression, based on file extension */
-            size_t const l = strlen(input_filename);
-            if (!strcmp(input_filename+(l-4), LZ4_EXTENSION)) decode=1;
+        if (mode == om_auto) {  /* auto-determine compression or decompression, based on file extension */
+            size_t const inSize = strlen(input_filename);
+            size_t const extSize = strlen(LZ4_EXTENSION);
+            if (!strcmp(input_filename+(inSize-extSize), LZ4_EXTENSION)) mode = om_decompress;
+            else mode = om_compress;
         }
-        if (!decode) {   /* compression to file */
+        if (mode == om_compress) {   /* compression to file */
             size_t const l = strlen(input_filename);
             dynNameSpace = (char*)calloc(1,l+5);
-			if (dynNameSpace==NULL) exit(1);
+            if (dynNameSpace==NULL) { perror(exeName); exit(1); }
             strcpy(dynNameSpace, input_filename);
             strcat(dynNameSpace, LZ4_EXTENSION);
             output_filename = dynNameSpace;
             DISPLAYLEVEL(2, "Compressed filename will be : %s \n", output_filename);
             break;
         }
-        /* decompression to file (automatic name will work only if input filename has correct format extension) */
-        {   size_t outl;
-            size_t inl = strlen(input_filename);
+        if (mode == om_decompress) {/* decompression to file (automatic name will work only if input filename has correct format extension) */
+            size_t outl;
+            size_t const inl = strlen(input_filename);
             dynNameSpace = (char*)calloc(1,inl+1);
+            if (dynNameSpace==NULL) { perror(exeName); exit(1); }
             strcpy(dynNameSpace, input_filename);
             outl = inl;
             if (inl>4)
                 while ((outl >= inl-4) && (input_filename[outl] ==  extension[outl-inl+4])) dynNameSpace[outl--]=0;
-            if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); badusage(); }
+            if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); badusage(exeName); }
             output_filename = dynNameSpace;
             DISPLAYLEVEL(2, "Decoding file %s \n", output_filename);
         }
+        break;
     }
 
     /* Check if output is defined as console; trigger an error in this case */
@@ -567,13 +571,12 @@ int main(int argc, const char** argv)
 
     /* IO Stream/File */
     LZ4IO_setNotificationLevel(displayLevel);
-    if (decode) {
+    if (mode == om_decompress) {
         if (multiple_inputs)
             operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, ifnIdx, LZ4_EXTENSION);
         else
             operationResult = DEFAULT_DECOMPRESSOR(input_filename, output_filename);
-    } else {
-        /* compression is default action */
+    } else {   /* compression is default action */
         if (legacy_format) {
             DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated) ! \n");
             LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel);
diff --git a/programs/lz4io.h b/programs/lz4io.h
index 45ae78d..315c805 100644
--- a/programs/lz4io.h
+++ b/programs/lz4io.h
@@ -29,7 +29,8 @@
   - The license of this source file is GPLv2.
 */
 
-#pragma once
+#ifndef LZ4IO_H_237902873
+#define LZ4IO_H_237902873
 
 /* ************************************************** */
 /* Special input/output values                        */
@@ -51,10 +52,10 @@ static char const nulmark[] = "/dev/null";
 int LZ4IO_compressFilename  (const char* input_filename, const char* output_filename, int compressionlevel);
 int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename);
 
-
 int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel);
 int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix);
 
+
 /* ************************************************** */
 /* ****************** Parameters ******************** */
 /* ************************************************** */
@@ -91,3 +92,6 @@ int LZ4IO_setSparseFile(int enable);
 int LZ4IO_setContentSize(int enable);
 
 void LZ4IO_setRemoveSrcFile(unsigned flag);
+
+
+#endif  /* LZ4IO_H_237902873 */
diff --git a/tests/.gitignore b/tests/.gitignore
index 8861e67..06891ac 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -7,3 +7,7 @@ fullbench
 fullbench32
 fuzzer
 fuzzer32
+fasttest
+
+# test artefacts
+tmp*
diff --git a/tests/Makefile b/tests/Makefile
index 6df6381..e6779d9 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -77,7 +77,7 @@ lz4c:
 	$(MAKE) -C $(PRGDIR) $@
 
 lz4c32:
-	$(MAKE) -C $(PRGDIR) $@ 
+	$(MAKE) -C $(PRGDIR) $@
 
 fullbench  : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o fullbench.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
@@ -221,22 +221,34 @@ test-lz4-basic: lz4 datagen
 	./datagen -g17M   | $(PRGDIR)/lz4 -9v    | $(PRGDIR)/lz4 -qt
 	./datagen -g33M   | $(PRGDIR)/lz4 --no-frame-crc | $(PRGDIR)/lz4 -t
 	./datagen -g256MB | $(PRGDIR)/lz4 -vqB4D | $(PRGDIR)/lz4 -t
+	@$(RM) tmp*
+
+test-lz4-hugefile: lz4 datagen
+	@echo "\n ---- test huge files compression/decompression ----"
 	./datagen -g6GB   | $(PRGDIR)/lz4 -vqB5D | $(PRGDIR)/lz4 -qt
-	./datagen -g6GB   | $(PRGDIR)/lz4 -vq9BD | $(PRGDIR)/lz4 -qt
+	./datagen -g6GB   | $(PRGDIR)/lz4 -vq5BD | $(PRGDIR)/lz4 -qt
 	@$(RM) tmp*
 
-test-lz4: lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-sparse test-lz4-contentSize test-lz4-frame-concatenation
-	@echo "\n ---- test pass-through ----"
+test-lz4-testmode: lz4 datagen
+	@echo "\n ---- test and pass-through ----"
 	./datagen | $(PRGDIR)/lz4 -t             && false || true
 	./datagen | $(PRGDIR)/lz4 -tf            && false || true
 	./datagen | $(PRGDIR)/lz4 -d  > $(VOID)  && false || true
 	./datagen | $(PRGDIR)/lz4 -df > $(VOID)
+	@echo "Hello World !" > tmp1
+	$(PRGDIR)/lz4 -dcf tmp1
+#	@echo "from underground..." > tmp2
+#	$(PRGDIR)/lz4 -dcfm tmp1 tmp2
 	@echo "\n ---- test cli ----"
 	$(PRGDIR)/lz4     file-does-not-exist    && false || true
 	$(PRGDIR)/lz4 -f  file-does-not-exist    && false || true
 	$(PRGDIR)/lz4 -fm file1-dne file2-dne    && false || true
 	$(PRGDIR)/lz4 -fm file1-dne file2-dne    && false || true
 
+test-lz4: lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-sparse \
+          test-lz4-frame-concatenation test-lz4-testmode test-lz4-contentSize \
+          test-lz4-hugefile
+
 test-lz4c: lz4c datagen
 	@echo "\n ---- test lz4c version ----"
 	./datagen -g256MB | $(PRGDIR)/lz4c -l -v    | $(PRGDIR)/lz4c   -t