diff options
author | Yann Collet <yann.collet.73@gmail.com> | 2015-03-22 12:42:00 (GMT) |
---|---|---|
committer | Yann Collet <yann.collet.73@gmail.com> | 2015-03-22 12:42:00 (GMT) |
commit | 7cf4e5c94136ee625d7225a72d4c38589a03ec5d (patch) | |
tree | 5775808a82e56f5318d7030bae2d4bf3e41058a2 /programs | |
parent | f02adc79389732177dca6fa21a3e716249aa63dd (diff) | |
download | lz4-7cf4e5c94136ee625d7225a72d4c38589a03ec5d.zip lz4-7cf4e5c94136ee625d7225a72d4c38589a03ec5d.tar.gz lz4-7cf4e5c94136ee625d7225a72d4c38589a03ec5d.tar.bz2 |
Updates tests & Man pages
Diffstat (limited to 'programs')
-rw-r--r-- | programs/Makefile | 110 | ||||
-rw-r--r-- | programs/fuzzer.c | 58 | ||||
-rw-r--r-- | programs/lz4.1 | 197 | ||||
-rw-r--r-- | programs/lz4cli.c | 21 |
4 files changed, 238 insertions, 148 deletions
diff --git a/programs/Makefile b/programs/Makefile index ae56131..f62adb1 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -95,7 +95,6 @@ frametest32: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xx datagen : datagen.c datagencli.c $(CC) $(FLAGS) $^ -o $@$(EXT) - clean: @rm -f core *.o *.test \ lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) \ @@ -113,24 +112,26 @@ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU)) install: lz4 lz4c @echo Installing binaries @install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ - @install -m 755 lz4$(EXT) $(DESTDIR)$(BINDIR)/lz4$(EXT) - @ln -sf lz4$(EXT) $(DESTDIR)$(BINDIR)/lz4cat - @ln -sf lz4$(EXT) $(DESTDIR)$(BINDIR)/unlz4 - @install -m 755 lz4c$(EXT) $(DESTDIR)$(BINDIR)/lz4c$(EXT) + @install -m 755 lz4 $(DESTDIR)$(BINDIR)/lz4 + @ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4cat + @ln -sf lz4 $(DESTDIR)$(BINDIR)/unlz4 + @install -m 755 lz4c $(DESTDIR)$(BINDIR)/lz4c @echo Installing man pages @install -m 644 lz4.1 $(DESTDIR)$(MANDIR)/lz4.1 - @install -m 644 lz4c.1 $(DESTDIR)$(MANDIR)/lz4c.1 - @install -m 644 lz4cat.1 $(DESTDIR)$(MANDIR)/lz4cat.1 + @ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4c.1 + @ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4cat.1 + @ln -sf lz4.1 $(DESTDIR)$(MANDIR)/unlz4.1 @echo lz4 installation completed uninstall: rm -f $(DESTDIR)$(BINDIR)/lz4cat rm -f $(DESTDIR)$(BINDIR)/unlz4 - [ -x $(DESTDIR)$(BINDIR)/lz4$(EXT) ] && rm -f $(DESTDIR)$(BINDIR)/lz4$(EXT) - [ -x $(DESTDIR)$(BINDIR)/lz4c$(EXT) ] && rm -f $(DESTDIR)$(BINDIR)/lz4c$(EXT) + [ -x $(DESTDIR)$(BINDIR)/lz4 ] && rm -f $(DESTDIR)$(BINDIR)/lz4 + [ -x $(DESTDIR)$(BINDIR)/lz4c ] && rm -f $(DESTDIR)$(BINDIR)/lz4c [ -f $(DESTDIR)$(MANDIR)/lz4.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4.1 - [ -f $(DESTDIR)$(MANDIR)/lz4c.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4c.1 - [ -f $(DESTDIR)$(MANDIR)/lz4cat.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4cat.1 + rm -f $(DESTDIR)$(MANDIR)/lz4c.1 + rm -f $(DESTDIR)$(MANDIR)/lz4cat.1 + rm -f $(DESTDIR)$(MANDIR)/unlz4.1 @echo lz4 programs successfully uninstalled test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-mem @@ -141,13 +142,37 @@ test-all: test test32 test-travis: $(TRAVIS_TARGET) -test-lz4: lz4 datagen - ./datagen -g16KB | ./lz4 -9 | ./lz4 -vdq > $(VOID) - ./datagen | ./lz4 | ./lz4 -vdq > $(VOID) - ./datagen -g6M -P100 | ./lz4 -9BD | ./lz4 -vdq > $(VOID) - ./datagen -g17M | ./lz4 -9v | ./lz4 -dq > $(VOID) - ./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -vdq > $(VOID) - ./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -vdq > $(VOID) +test-lz4-sparse: lz4 datagen + @echo ---- test sparse file support ---- + ./datagen -g50M -P100 | ./lz4 -B4D | ./lz4 -dv --sparse-support > tmpB4 + ./datagen -g50M -P100 | ./lz4 -B5D | ./lz4 -dv --sparse-support > tmpB5 + ./datagen -g50M -P100 | ./lz4 -B6D | ./lz4 -dv --sparse-support > tmpB6 + ./datagen -g50M -P100 | ./lz4 -B7D | ./lz4 -dv --sparse-support > tmpB7 + ls -ls tmp* + ./datagen -g50M -P100 | diff -s - tmpB4 + ./datagen -g50M -P100 | diff -s - tmpB5 + ./datagen -g50M -P100 | diff -s - tmpB6 + ./datagen -g50M -P100 | diff -s - tmpB7 + ./datagen -s1 -g1200007 -P100 | ./lz4 | ./lz4 -dv --sparse-support > tmpOdd # Odd size file (to not finish on an exact nb of blocks) + ./datagen -s1 -g1200007 -P100 | diff -s - tmpOdd + ls -ls tmpOdd + @rm tmp* + +test-lz4-contentSize: lz4 datagen + @echo ---- test original size support ---- + ./datagen -g15M | ./lz4 -v | ./lz4 -t + ./datagen -g15M | ./lz4 -v --frame-content-size | ./lz4 -t + +test-lz4: lz4 datagen test-lz4-sparse test-lz4-contentSize + @echo ---- test lz4 basic compression/decompression ---- + ./datagen -g16KB | ./lz4 -9 | ./lz4 -t + ./datagen | ./lz4 | ./lz4 -t + ./datagen -g6M -P99 | ./lz4 -9BD | ./lz4 -t + ./datagen -g17M | ./lz4 -9v | ./lz4 -tq + ./datagen -g33M | ./lz4 --no-frame-crc | ./lz4 -t + ./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -t + ./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -t + ./datagen -g6GB | ./lz4 -vq9BD | ./lz4 -t @echo ---- test frame concatenation ---- @echo -n > empty.test @echo hi > nonempty.test @@ -166,35 +191,21 @@ test-lz4: lz4 datagen ./lz4 -f -m file1 file2 file3 ls -l file* @rm file1 file2 file3 file1.lz4 file2.lz4 file3.lz4 - @echo ---- test sparse file support ---- - ./datagen -g50M -P100 | ./lz4 -B4D | ./lz4 -dvX > tmpB4 - ./datagen -g50M -P100 | ./lz4 -B5D | ./lz4 -dvX > tmpB5 - ./datagen -g50M -P100 | ./lz4 -B6D | ./lz4 -dvX > tmpB6 - ./datagen -g50M -P100 | ./lz4 -B7D | ./lz4 -dvX > tmpB7 - ls -ls tmp* - ./datagen -g50M -P100 | diff -s - tmpB4 - ./datagen -g50M -P100 | diff -s - tmpB5 - ./datagen -g50M -P100 | diff -s - tmpB6 - ./datagen -g50M -P100 | diff -s - tmpB7 - ./datagen -s1 -g1200007 -P100 | ./lz4 | ./lz4 -dvX > tmpOdd # Odd size file (to not finish on an exact nb of blocks) - ./datagen -s1 -g1200007 -P100 | diff -s - tmpOdd - ls -ls tmpOdd - @rm tmp* @echo ---- test pass-through ---- - ./datagen | ./lz4 -dfvq > /dev/null - + ./datagen | ./lz4 -tf test-lz4c: lz4c datagen - ./datagen -g256MB | ./lz4c -l -v | ./lz4c -vdq > $(VOID) - -test-lz4c32: lz4 lz4c32 lz4 datagen - ./datagen -g16KB | ./lz4c32 -9 | ./lz4c32 -vdq > $(VOID) - ./datagen -g16KB | ./lz4c32 -9 | ./lz4 -vdq > $(VOID) - ./datagen | ./lz4c32 | ./lz4c32 -vdq > $(VOID) - ./datagen | ./lz4c32 | ./lz4 -vdq > $(VOID) - ./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4c32 -vdq > $(VOID) - ./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4 -vdq > $(VOID) - ./datagen -g6GB | ./lz4c32 -vqB5D | ./lz4c32 -vdq > $(VOID) + ./datagen -g256MB | ./lz4c -l -v | ./lz4c -t + +test-lz4c32: lz4 lz4c32 datagen + ./datagen -g16KB | ./lz4c32 -9 | ./lz4c32 -t + ./datagen -g16KB | ./lz4c32 -9 | ./lz4 -t + ./datagen | ./lz4c32 | ./lz4c32 -t + ./datagen | ./lz4c32 | ./lz4 -t + ./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4c32 -t + ./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4 -t + ./datagen -g6GB | ./lz4c32 -vqB5D | ./lz4c32 -t + ./datagen -g6GB | ./lz4c32 -vq9BD | ./lz4 -t test-fullbench: fullbench ./fullbench --no-prompt $(TEST_FILES) @@ -215,14 +226,15 @@ test-frametest32: frametest32 ./frametest32 test-mem: lz4 datagen fuzzer frametest - valgrind --leak-check=yes ./datagen -g50M > /dev/null + valgrind --leak-check=yes ./datagen -g50M > $(VOID) ./datagen -g16KB > tmp - valgrind --leak-check=yes ./lz4 -9 -BD -f tmp /dev/null + valgrind --leak-check=yes ./lz4 -9 -BD -f tmp $(VOID) ./datagen -g16MB > tmp - valgrind --leak-check=yes ./lz4 -9 -B5D -f tmp /dev/null + valgrind --leak-check=yes ./lz4 -9 -B5D -f tmp tmp2 ./datagen -g256MB > tmp - valgrind --leak-check=yes ./lz4 -B4D -f -vq tmp /dev/null - rm tmp + valgrind --leak-check=yes ./lz4 -t tmp2 + valgrind --leak-check=yes ./lz4 -B4D -f -vq tmp $(VOID) + rm tmp* valgrind --leak-check=yes ./fuzzer -i64 -t1 valgrind --leak-check=yes ./frametest -i256 diff --git a/programs/fuzzer.c b/programs/fuzzer.c index d126da2..3d3cf8e 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -989,7 +989,7 @@ static void FUZ_unitTests(void) crcNew = XXH64_digest(&xxhNew); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); - // prepare next message + /* prepare next message */ dNext += messageSize; totalMessageSize += messageSize; messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1; @@ -997,62 +997,6 @@ static void FUZ_unitTests(void) if (dNext + messageSize > dBufferSize) dNext = 0; } } - - // long stream test ; Warning : very long test ! - if (1) - { - XXH64_state_t crcOrigState; - XXH64_state_t crcNewState; - const U64 totalTestSize = 6ULL << 30; - U64 totalTestDone = 0; - size_t oldStart = 0; - size_t oldSize = 0; - U32 segNb = 1; - - DISPLAY("Long HC streaming test (%u MB)\n", (U32)(totalTestSize >> 20)); - LZ4_resetStreamHC(&sHC, 0); - - XXH64_reset(&crcOrigState, 0); - XXH64_reset(&crcNewState, 0); - - while (totalTestDone < totalTestSize) - { - size_t testSize = (FUZ_rand(&randState) & 65535) + 1; - size_t testStart = FUZ_rand(&randState) & 65535; - - FUZ_displayUpdate((U32)(totalTestDone >> 20)); - - if (testStart == oldStart + oldSize) // Corner case not covered by this test (LZ4_decompress_safe_usingDict() limitation) - testStart++; - - XXH64_update(&crcOrigState, testInput + testStart, testSize); - crcOrig = XXH64_digest(&crcOrigState); - - result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + testStart, testCompressed, (int)testSize, LZ4_compressBound((int)testSize)); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); - - result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, (int)testSize, testInput + oldStart, (int)oldSize); - FUZ_CHECKTEST(result!=(int)testSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %u failed", segNb); - - XXH64_update(&crcNewState, testVerify, testSize); - crcNew = XXH64_digest(&crcNewState); - if (crcOrig!=crcNew) - { - size_t c=0; - while (testVerify[c] == testInput[testStart+c]) c++; - DISPLAY("Bad decompression at %u / %u \n", (U32)c, (U32)testSize); - } - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() part %u corruption", segNb); - - oldStart = testStart; - oldSize = testSize; - totalTestDone += testSize; - - segNb ++; - } - - DISPLAY("\r"); - } } printf("All unit tests completed successfully \n"); diff --git a/programs/lz4.1 b/programs/lz4.1 index 6ae8d3c..8bab7a3 100644 --- a/programs/lz4.1 +++ b/programs/lz4.1 @@ -1,79 +1,202 @@ \" \" lz4.1: This is a manual page for 'lz4' program. This file is part of the \" lz4 <https://code.google.com/p/lz4/> project. +\" Author: Yann Collet \" \" No hyphenation .hy 0 .nr HY 0 -.TH lz4 "1" "2014-02-27" "lz4" "User Commands" +.TH lz4 "1" "2015-03-21" "lz4" "User Commands" .SH NAME -\fBlz4\fR - Extremely fast compression algorithm +\fBlz4, unlz4, lz4cat\fR \- Compress or decompress .lz4 files .SH SYNOPSIS .TP 5 \fBlz4\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE> +.PP +.B unlz4 +is equivalent to +.BR "lz4 \-d" +.br +.B lz4cat +is equivalent to +.BR "lz4 \-dc" +.br +.PP +When writing scripts that need to decompress files, +it is recommended to always use the name +.B lz4 +with appropriate arguments +.RB ( "lz4 \-d" +or +.BR "lz4 \-dc" ) +instead of the names +.B unlz4 +and +.BR lz4cat . + .SH DESCRIPTION .PP -\fBlz4\fR is an extremely fast lossless compression algorithm. It is based on -the \fBLZ77\fR family of compression scheme. At the compression speed of 400 -MB/s per core, \fBlz4\fR is also scalable with multi-core CPUs. It features -an extremely fast decoder, with speed in multiple GB/s per core, typically -reaching the RAM speed limits on multi-core systems. \fBlz4\fR supports -following options +\fBlz4\fR is an extremely fast lossless compression algorithm, +based on \fBbyte-aligned LZ77\fR family of compression scheme. +\fBlz4\fR offers compression speeds of 400 MB/s per core, linearly scalable with multi-core CPUs. +It features an extremely fast decoder, with speed in multiple GB/s per core, +typically reaching RAM speed limit on multi-core systems. +.B lz4 +supports a command line syntax similar to +.BR gzip (1). +The native file format is the +.B .lz4 +format. + +.SS "Concatenation of .lz4 files" +It is possible to concatenate +.B .lz4 +files as is. +.B lz4 +will decompress such files as if they were a single +.B .lz4 +file. +.PP .SH OPTIONS +. +.SS "Short command concatenation" +In some cases, some options can be expressed using short command +.B "-x" +or long command +.B "--long-word" . +Short commands can be concatenated together. For example, +.B "-d -c" +is equivalent to +.B "-dc" . +Long commands cannot be concatenated. +They must be clearly separated by a space. +. +.SS "Operation mode" +If multiple operation mode options are given, +the last one takes effect. +.TP +.BR \-z ", " \-\-compress +Compress. +This is the default operation mode +when no operation mode option is specified , +no other operation mode is implied from the command name +(for example, +.B unlz4 +implies +.B \-\-decompress ), +nor from the input file name +(for example, a file extension +.B .lz4 +implies +.B \-\-decompress +by default). +.B -z +can also be used to force compression of an already compressed +.B .lz4 +file. +.TP +.BR \-d ", " \-\-decompress ", " \-\-uncompress +Decompress. +.B --decompress +is also the default operation when the input filename has an +.B .lz4 +extensionq +.TP +.BR \-t ", " \-\-test +Test the integrity of compressed +.B .lz4 +files. +The decompressed data is discarded. +No files are created or removed. +. +.SS "Operation modifiers" .TP .B \-1 fast compression (default) .TP .B \-9 high compression + .TP -.B \-d - decompression -.TP -.B \-f - overwrite output without prompting +.BR \-f ", " --force + This option has several effects: +.RS +.IP \(bu 3 +If the target file already exists, +overwrite it without prompting. +.IP \(bu 3 +When used with +.B \-\-decompress +and +.B lz4 +cannot recognize the type of the source file, +copy the source file as is to standard output. +This allows +.B lz4cat +.B \-\-force +to be used like +.BR cat (1) +for files that have not been compressed with +.BR lz4 . +.RE + .TP -.B \-h/\-H - display help/long help and exit +.BR \-c ", " \--stdout ", " \--to-stdout + force write to standard output, even if it is the console + .TP -.B \-V - display Version number and exit +.BR \-m + Multiple file names. + By default, the second filename is used as the output filename for the compressed file. + With +.B -m +, you can specify any number of input filenames, each of them will be compressed +with the resulting compressed file named +.B filename.lz4 +. + .TP -.B \-v - verbose mode +.B \-B# + block size [4-7](default : 7) + B4= 64KB ; B5= 256KB ; B6= 1MB ; B7= 4MB .TP -.B \-q - suppress warnings; specify twice to suppress errors too +.B \-BD + block dependency (improve compression ratio) .TP -.B \-c - force write to standard output, even if it is the console +.B \--no-frame-crc + disable stream checksum (default:enabled) .TP -.B \-t - test compressed file integrity +.B \--frame-content-size + compressed frame includes original size (default:not present) .TP -.B \-z - force compression +.B \--sparse-support + enable sparse file (default:disabled)(experimental) .TP .B \-l use Legacy format (useful for Linux Kernel compression) +. +.SS "Other options" .TP -.B \-B# - block size [4-7](default : 7) - B4= 64KB ; B5= 256KB ; B6= 1MB ; B7= 4MB +.BR \-v ", " --verbose + verbose mode .TP -.B \-BD - block dependency (improve compression ratio) +.BR \-q ", " --quiet + suppress warnings; specify twice to suppress errors too +.TP +.B \-h/\-H + display help/long help and exit .TP -.B \-BX - enable block checksum (default:disabled) +.BR \-V ", " \--version + display Version number and exit .TP -.B \-Sx - disable stream checksum (default:enabled) +.BR \-k ", " \--keep + Don't delete source file. +This is default behavior anyway, so this option is just for compatibility with gzip/xz. .TP .B \-b benchmark file(s) @@ -82,7 +205,7 @@ following options iteration loops [1-9](default : 3), benchmark mode only .SH BUGS -Report bugs at:- https://code.google.com/p/lz4/ +Report bugs at: https://github.com/Cyan4973/lz4 .SH AUTHOR Yann Collet diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 0f914b4..7fe5c3f 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -154,7 +154,6 @@ static int usage(void) DISPLAY( " -z : force compression\n"); DISPLAY( " -f : overwrite output without prompting \n"); DISPLAY( " -h/-H : display help/long help and exit\n"); - DISPLAY( " -m : multiple input files (implies automatic output filenames)\n"); return 0; } @@ -169,6 +168,7 @@ static int usage_advanced(void) DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); DISPLAY( " -c : force write to standard output, even if it is the console\n"); DISPLAY( " -t : test compressed file integrity\n"); + DISPLAY( " -m : multiple input files (implies automatic output filenames)\n"); DISPLAY( " -l : compress using Legacy format (Linux kernel compression)\n"); DISPLAY( " -B# : Block size [4-7](default : 7)\n"); DISPLAY( " -BD : Block dependency (improve compression ratio)\n"); @@ -291,12 +291,23 @@ int main(int argc, char** argv) if(!argument) continue; /* Protection if argument empty */ - /* long options (--****) */ - if (!strcmp(argument, "--sparse-support")) { LZ4IO_setSparseFile(1); continue; } + /* long commands (--long-word) */ + if (!strcmp(argument, "--compress")) { forceCompress = 1; continue; } + if (!strcmp(argument, "--decompress")) { decode = 1; continue; } + if (!strcmp(argument, "--uncompress")) { decode = 1; continue; } + if (!strcmp(argument, "--test")) { decode = 1; LZ4IO_setOverwrite(1); output_filename=nulmark; continue; } + if (!strcmp(argument, "--force")) { LZ4IO_setOverwrite(1); continue; } + if (!strcmp(argument, "--stdout")) { forceStdout=1; output_filename=stdoutmark; displayLevel=1; continue; } + if (!strcmp(argument, "--to-stdout")) { forceStdout=1; output_filename=stdoutmark; displayLevel=1; continue; } if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(0); continue; } if (!strcmp(argument, "--frame-content-size")) { continue; } + if (!strcmp(argument, "--sparse-support")) { LZ4IO_setSparseFile(1); continue; } + if (!strcmp(argument, "--verbose")) { displayLevel=4; continue; } + if (!strcmp(argument, "--quiet")) { if (displayLevel) displayLevel--; continue; } + if (!strcmp(argument, "--version")) { DISPLAY(WELCOME_MESSAGE); return 0; } + if (!strcmp(argument, "--keep")) { continue; } /* keep source file (default anyway, so useless) (for xz/lzma compatibility) */ - /* Decode command (note : aggregated commands are allowed) */ + /* Short commands (note : aggregated short commands are allowed) */ if (argument[0]=='-') { /* '-' means stdin/stdout */ @@ -360,7 +371,7 @@ int main(int argc, char** argv) case 'v': displayLevel=4; break; /* Quiet mode */ - case 'q': displayLevel--; break; + case 'q': if (displayLevel) displayLevel--; break; /* keep source file (default anyway, so useless) (for xz/lzma compatibility) */ case 'k': break; |