From acc48a3f85e5c97a102999544248ee7e51cc419b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 9 Nov 2016 14:36:42 -0800 Subject: fixed lz4cat changed lz4-test order --- NEWS | 2 +- programs/lz4.1 | 2 +- programs/lz4cli.c | 117 ++++++++++++++++++++++++++++-------------------------- programs/lz4io.h | 8 +++- tests/.gitignore | 4 ++ tests/Makefile | 20 ++++++++-- 6 files changed, 88 insertions(+), 65 deletions(-) diff --git a/NEWS b/NEWS index 8a74594..3257fd5 100644 --- a/NEWS +++ b/NEWS @@ -8,13 +8,13 @@ Fix : better ratio on 64-bits big-endian targets Improved cmake build script, by Evan Nemerson New liblz4-dll project, by Przemyslaw Skibinki Makefile: Generates object files (*.o) for faster (re)compilation on low power systems -Added : Debianization, by Evgeniy Polyakov cli : new : --rm command cli : new : file attributes are preserved, by Przemyslaw Skibinki cli : fix : crash on some invalid inputs cli : fix : -t correctly validates lz4-compressed files, by Nick Terrell cli : fix : detects and reports fread() errors, thanks to Hiroshi Fujishima report #243 cli : bench : new : -r recursive mode +Added : Debianization, by Evgeniy Polyakov r131 New : Dos/DJGPP target, thanks to Louis Santillan (#114) diff --git a/programs/lz4.1 b/programs/lz4.1 index 529d36a..5a28a83 100644 --- a/programs/lz4.1 +++ b/programs/lz4.1 @@ -22,7 +22,7 @@ is equivalent to .br .B lz4cat is equivalent to -.BR "lz4 \-dc" +.BR "lz4 \-dcf" .br .PP When writing scripts that need to decompress files, diff --git a/programs/lz4cli.c b/programs/lz4cli.c index def7894..fdfa8f2 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -99,12 +99,6 @@ static unsigned displayLevel = 2; /* 0 : no display ; 1: errors only ; 2 : dow /*-************************************ -* Local Variables -***************************************/ -static const char* programName; - - -/*-************************************ * Exceptions ***************************************/ #define DEBUG 0 @@ -133,10 +127,10 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output /*-*************************** * Functions *****************************/ -static int usage(void) +static int usage(const char* exeName) { DISPLAY( "Usage :\n"); - DISPLAY( " %s [arg] [input] [output]\n", programName); + DISPLAY( " %s [arg] [input] [output]\n", exeName); DISPLAY( "\n"); DISPLAY( "input : a filename\n"); DISPLAY( " with no FILE, or when FILE is - or %s, read standard input\n", stdinmark); @@ -151,10 +145,10 @@ static int usage(void) return 0; } -static int usage_advanced(void) +static int usage_advanced(const char* exeName) { DISPLAY(WELCOME_MESSAGE); - usage(); + usage(exeName); DISPLAY( "\n"); DISPLAY( "Advanced arguments :\n"); DISPLAY( " -V : display Version number and exit\n"); @@ -188,9 +182,9 @@ static int usage_advanced(void) return 0; } -static int usage_longhelp(void) +static int usage_longhelp(const char* exeName) { - usage_advanced(); + usage_advanced(exeName); DISPLAY( "\n"); DISPLAY( "****************************\n"); DISPLAY( "***** Advanced comment *****\n"); @@ -216,43 +210,43 @@ static int usage_longhelp(void) DISPLAY( "stdin, stdout and the console : \n"); DISPLAY( "--------------------------------\n"); DISPLAY( "To protect the console from binary flooding (bad argument mistake)\n"); - DISPLAY( "%s will refuse to read from console, or write to console \n", programName); + DISPLAY( "%s will refuse to read from console, or write to console \n", exeName); DISPLAY( "except if '-c' command is specified, to force output to console \n"); DISPLAY( "\n"); DISPLAY( "Simple example :\n"); DISPLAY( "----------------\n"); DISPLAY( "1 : compress 'filename' fast, using default output name 'filename.lz4'\n"); - DISPLAY( " %s filename\n", programName); + DISPLAY( " %s filename\n", exeName); DISPLAY( "\n"); DISPLAY( "Short arguments can be aggregated. For example :\n"); DISPLAY( "----------------------------------\n"); DISPLAY( "2 : compress 'filename' in high compression mode, overwrite output if exists\n"); - DISPLAY( " %s -9 -f filename \n", programName); + DISPLAY( " %s -9 -f filename \n", exeName); DISPLAY( " is equivalent to :\n"); - DISPLAY( " %s -9f filename \n", programName); + DISPLAY( " %s -9f filename \n", exeName); DISPLAY( "\n"); - DISPLAY( "%s can be used in 'pure pipe mode'. For example :\n", programName); + DISPLAY( "%s can be used in 'pure pipe mode'. For example :\n", exeName); DISPLAY( "-------------------------------------\n"); DISPLAY( "3 : compress data stream from 'generator', send result to 'consumer'\n"); - DISPLAY( " generator | %s | consumer \n", programName); + DISPLAY( " generator | %s | consumer \n", exeName); #if defined(ENABLE_LZ4C_LEGACY_OPTIONS) DISPLAY( "\n"); DISPLAY( "***** Warning *****\n"); DISPLAY( "Legacy arguments take precedence. Therefore : \n"); DISPLAY( "---------------------------------\n"); - DISPLAY( " %s -hc filename\n", programName); + DISPLAY( " %s -hc filename\n", exeName); DISPLAY( "means 'compress filename in high compression mode'\n"); DISPLAY( "It is not equivalent to :\n"); - DISPLAY( " %s -h -c filename\n", programName); + DISPLAY( " %s -h -c filename\n", exeName); DISPLAY( "which would display help text and exit\n"); #endif /* ENABLE_LZ4C_LEGACY_OPTIONS */ return 0; } -static int badusage(void) +static int badusage(const char* exeName) { DISPLAYLEVEL(1, "Incorrect parameters\n"); - if (displayLevel >= 1) usage(); + if (displayLevel >= 1) usage(exeName); exit(1); } @@ -276,28 +270,29 @@ static unsigned readU32FromChar(const char** stringPtr) return result; } +typedef enum { om_auto, om_compress, om_decompress, om_test, om_bench } operationMode_e; int main(int argc, const char** argv) { int i, cLevel=1, cLevelLast=1, - decode=0, - bench=0, legacy_format=0, forceStdout=0, forceCompress=0, main_pause=0, multiple_inputs=0, operationResult=0; + operationMode_e mode = om_auto; const char* input_filename=0; const char* output_filename=0; char* dynNameSpace=0; const char** inFileNames = NULL; unsigned ifnIdx=0; - char nullOutput[] = NULL_OUTPUT; - char extension[] = LZ4_EXTENSION; - int blockSize; + const char nullOutput[] = NULL_OUTPUT; + const char extension[] = LZ4_EXTENSION; + int blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT); + const char* const exeName = argv[0]; #ifdef UTIL_HAS_CREATEFILELIST const char** extendedFileList = NULL; char* fileNamesBuf = NULL; @@ -305,13 +300,11 @@ int main(int argc, const char** argv) #endif /* Init */ - programName = argv[0]; LZ4IO_setOverwrite(0); - blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT); /* lz4cat predefined behavior */ - if (!strcmp(programName, LZ4CAT)) { decode=1; forceStdout=1; output_filename=stdoutmark; displayLevel=1; } - if (!strcmp(programName, UNLZ4)) { decode=1; } + if (!strcmp(exeName, LZ4CAT)) { mode = om_decompress; LZ4IO_setOverwrite(1); forceStdout=1; output_filename=stdoutmark; displayLevel=1; } + if (!strcmp(exeName, UNLZ4)) { mode = om_decompress; } /* command switches */ for(i=1; i>10); + if (mode == om_compress) DISPLAYLEVEL(4, "Blocks size : %i KB\n", blockSize>>10); /* No input filename ==> use stdin */ if (multiple_inputs) { @@ -517,42 +510,53 @@ int main(int argc, const char** argv) exit(1); } - /* Check if benchmark is selected */ - if (bench) { + /* benchmark and test modes */ + if (mode == om_bench) { BMK_setNotificationLevel(displayLevel); operationResult = BMK_benchFiles(inFileNames, ifnIdx, cLevel, cLevelLast); goto _cleanup; } + if (mode == om_test) { + LZ4IO_setTestMode(1); + output_filename=nulmark; + mode = om_decompress; /* defer to decompress */ + } + + /* compress or decompress */ /* No output filename ==> try to select one automatically (when possible) */ while (!output_filename) { if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; break; } /* Default to stdout whenever possible (i.e. not a console) */ - if ((!decode) && !(forceCompress)) { /* auto-determine compression or decompression, based on file extension */ - size_t const l = strlen(input_filename); - if (!strcmp(input_filename+(l-4), LZ4_EXTENSION)) decode=1; + if (mode == om_auto) { /* auto-determine compression or decompression, based on file extension */ + size_t const inSize = strlen(input_filename); + size_t const extSize = strlen(LZ4_EXTENSION); + if (!strcmp(input_filename+(inSize-extSize), LZ4_EXTENSION)) mode = om_decompress; + else mode = om_compress; } - if (!decode) { /* compression to file */ + if (mode == om_compress) { /* compression to file */ size_t const l = strlen(input_filename); dynNameSpace = (char*)calloc(1,l+5); - if (dynNameSpace==NULL) exit(1); + if (dynNameSpace==NULL) { perror(exeName); exit(1); } strcpy(dynNameSpace, input_filename); strcat(dynNameSpace, LZ4_EXTENSION); output_filename = dynNameSpace; DISPLAYLEVEL(2, "Compressed filename will be : %s \n", output_filename); break; } - /* decompression to file (automatic name will work only if input filename has correct format extension) */ - { size_t outl; - size_t inl = strlen(input_filename); + if (mode == om_decompress) {/* decompression to file (automatic name will work only if input filename has correct format extension) */ + size_t outl; + size_t const inl = strlen(input_filename); dynNameSpace = (char*)calloc(1,inl+1); + if (dynNameSpace==NULL) { perror(exeName); exit(1); } strcpy(dynNameSpace, input_filename); outl = inl; if (inl>4) while ((outl >= inl-4) && (input_filename[outl] == extension[outl-inl+4])) dynNameSpace[outl--]=0; - if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); badusage(); } + if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); badusage(exeName); } output_filename = dynNameSpace; DISPLAYLEVEL(2, "Decoding file %s \n", output_filename); } + break; } /* Check if output is defined as console; trigger an error in this case */ @@ -567,13 +571,12 @@ int main(int argc, const char** argv) /* IO Stream/File */ LZ4IO_setNotificationLevel(displayLevel); - if (decode) { + if (mode == om_decompress) { if (multiple_inputs) operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, ifnIdx, LZ4_EXTENSION); else operationResult = DEFAULT_DECOMPRESSOR(input_filename, output_filename); - } else { - /* compression is default action */ + } else { /* compression is default action */ if (legacy_format) { DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated) ! \n"); LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel); diff --git a/programs/lz4io.h b/programs/lz4io.h index 45ae78d..315c805 100644 --- a/programs/lz4io.h +++ b/programs/lz4io.h @@ -29,7 +29,8 @@ - The license of this source file is GPLv2. */ -#pragma once +#ifndef LZ4IO_H_237902873 +#define LZ4IO_H_237902873 /* ************************************************** */ /* Special input/output values */ @@ -51,10 +52,10 @@ static char const nulmark[] = "/dev/null"; int LZ4IO_compressFilename (const char* input_filename, const char* output_filename, int compressionlevel); int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename); - int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel); int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix); + /* ************************************************** */ /* ****************** Parameters ******************** */ /* ************************************************** */ @@ -91,3 +92,6 @@ int LZ4IO_setSparseFile(int enable); int LZ4IO_setContentSize(int enable); void LZ4IO_setRemoveSrcFile(unsigned flag); + + +#endif /* LZ4IO_H_237902873 */ diff --git a/tests/.gitignore b/tests/.gitignore index 8861e67..06891ac 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -7,3 +7,7 @@ fullbench fullbench32 fuzzer fuzzer32 +fasttest + +# test artefacts +tmp* diff --git a/tests/Makefile b/tests/Makefile index 6df6381..e6779d9 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -77,7 +77,7 @@ lz4c: $(MAKE) -C $(PRGDIR) $@ lz4c32: - $(MAKE) -C $(PRGDIR) $@ + $(MAKE) -C $(PRGDIR) $@ fullbench : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o fullbench.c $(CC) $(FLAGS) $^ -o $@$(EXT) @@ -221,22 +221,34 @@ test-lz4-basic: lz4 datagen ./datagen -g17M | $(PRGDIR)/lz4 -9v | $(PRGDIR)/lz4 -qt ./datagen -g33M | $(PRGDIR)/lz4 --no-frame-crc | $(PRGDIR)/lz4 -t ./datagen -g256MB | $(PRGDIR)/lz4 -vqB4D | $(PRGDIR)/lz4 -t + @$(RM) tmp* + +test-lz4-hugefile: lz4 datagen + @echo "\n ---- test huge files compression/decompression ----" ./datagen -g6GB | $(PRGDIR)/lz4 -vqB5D | $(PRGDIR)/lz4 -qt - ./datagen -g6GB | $(PRGDIR)/lz4 -vq9BD | $(PRGDIR)/lz4 -qt + ./datagen -g6GB | $(PRGDIR)/lz4 -vq5BD | $(PRGDIR)/lz4 -qt @$(RM) tmp* -test-lz4: lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-sparse test-lz4-contentSize test-lz4-frame-concatenation - @echo "\n ---- test pass-through ----" +test-lz4-testmode: lz4 datagen + @echo "\n ---- test and pass-through ----" ./datagen | $(PRGDIR)/lz4 -t && false || true ./datagen | $(PRGDIR)/lz4 -tf && false || true ./datagen | $(PRGDIR)/lz4 -d > $(VOID) && false || true ./datagen | $(PRGDIR)/lz4 -df > $(VOID) + @echo "Hello World !" > tmp1 + $(PRGDIR)/lz4 -dcf tmp1 +# @echo "from underground..." > tmp2 +# $(PRGDIR)/lz4 -dcfm tmp1 tmp2 @echo "\n ---- test cli ----" $(PRGDIR)/lz4 file-does-not-exist && false || true $(PRGDIR)/lz4 -f file-does-not-exist && false || true $(PRGDIR)/lz4 -fm file1-dne file2-dne && false || true $(PRGDIR)/lz4 -fm file1-dne file2-dne && false || true +test-lz4: lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-sparse \ + test-lz4-frame-concatenation test-lz4-testmode test-lz4-contentSize \ + test-lz4-hugefile + test-lz4c: lz4c datagen @echo "\n ---- test lz4c version ----" ./datagen -g256MB | $(PRGDIR)/lz4c -l -v | $(PRGDIR)/lz4c -t -- cgit v0.12