diff options
| -rw-r--r-- | .travis.yml | 6 | ||||
| -rw-r--r-- | LICENSE | 2 | ||||
| -rw-r--r-- | Makefile | 10 | ||||
| -rw-r--r-- | NEWS | 6 | ||||
| -rw-r--r-- | README.md | 49 | ||||
| -rw-r--r-- | lz4.h | 2 | ||||
| -rw-r--r-- | lz4hc.h | 2 | ||||
| -rw-r--r-- | programs/Makefile | 25 | ||||
| -rw-r--r-- | programs/bench.c | 2 | ||||
| -rw-r--r-- | programs/bench.h | 2 | ||||
| -rw-r--r-- | programs/fullbench.c | 24 | ||||
| -rw-r--r-- | programs/fuzzer.c | 25 | ||||
| -rw-r--r-- | programs/lz4c.1 | 33 | ||||
| -rw-r--r-- | programs/lz4cat.1 | 34 | ||||
| -rw-r--r-- | programs/lz4cli.c | 6 | ||||
| -rw-r--r-- | programs/lz4io.c | 2 | ||||
| -rw-r--r-- | programs/lz4io.h | 2 |
17 files changed, 204 insertions, 28 deletions
diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..695c19f --- /dev/null +++ b/.travis.yml @@ -0,0 +1,6 @@ +language: c +compiler: gcc +script: make test +before_install: + - sudo apt-get update -qq + - sudo apt-get install -qq gcc-multilib @@ -1,5 +1,5 @@ LZ4 Library -Copyright (c) 2013, Yann Collet +Copyright (c) 2011-2014, Yann Collet All rights reserved. Redistribution and use in source and binary forms, with or without modification, @@ -30,7 +30,7 @@ # - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c # ################################################################ -export RELEASE=r116 +export RELEASE=r117 LIBVER_MAJOR=1 LIBVER_MINOR=0 LIBVER_PATCH=0 @@ -69,13 +69,14 @@ else endif TEXT = lz4.c lz4.h lz4hc.c lz4hc.h \ - lz4_format_description.txt Makefile NEWS LICENSE \ + lz4_format_description.txt Makefile NEWS LICENSE README.md \ cmake_unofficial/CMakeLists.txt \ $(PRGDIR)/fullbench.c $(PRGDIR)/fuzzer.c $(PRGDIR)/lz4cli.c \ $(PRGDIR)/lz4io.c $(PRGDIR)/lz4io.h \ $(PRGDIR)/bench.c $(PRGDIR)/bench.h \ $(PRGDIR)/xxhash.c $(PRGDIR)/xxhash.h \ - $(PRGDIR)/lz4.1 $(PRGDIR)/Makefile $(PRGDIR)/COPYING + $(PRGDIR)/lz4.1 $(PRGDIR)/lz4c.1 $(PRGDIR)/lz4cat.1 \ + $(PRGDIR)/Makefile $(PRGDIR)/COPYING NONTEXT = LZ4_Streaming_Format.odt SOURCES = $(TEXT) $(NONTEXT) @@ -144,4 +145,7 @@ dist: clean @sha1sum $(DISTRIBNAME) > $(DISTRIBNAME).sha1 @echo Distribution $(DISTRIBNAME) built +test: lz4programs + @cd $(PRGDIR); $(MAKE) -e $@ + endif @@ -1,3 +1,9 @@ +r117: +Added : man pages for lz4c and lz4cat +Added : automated tests on Travis, thanks to Takayuki Matsuoka ! +fix : block-dependency command line (issue 127) +fix : lz4fullbench (issue 128) + r116: hotfix (issue 124 & 125) diff --git a/README.md b/README.md new file mode 100644 index 0000000..98ba7c6 --- /dev/null +++ b/README.md @@ -0,0 +1,49 @@ +LZ4 - Extremely fast compression +================================ + +LZ4 is lossless compression algorithm, providing compression speed at 400 MB/s per core, scalable with multi-cores CPU. It also features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. +A high compression derivative, called LZ4_HC, is also provided. It trades CPU time for compression ratio. + +This is an official mirror of LZ4 project, [hosted on Google Code](http://code.google.com/p/lz4/). +The intention is to offer github's capabilities to lz4 users, such as cloning, branch, or source download. + +The "master" branch will reflect, the status of lz4 at its official homepage. Other branches will also exist, typically to fix some open issues or new requirements, and be available for testing before merge into master. + + +Benchmarks +------------------------- + +The benchmark uses the [Open-Source Benchmark program by m^2 (v0.14.2)](http://encode.ru/threads/1371-Filesystem-benchmark?p=33548&viewfull=1#post33548) compiled with GCC v4.6.1 on Linux Ubuntu 64-bits v11.10, +The reference system uses a Core i5-3340M @2.7GHz. +Benchmark evaluates the compression of reference [Silesia Corpus](http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia) in single-thread mode. + +<table> + <tr> + <th>Compressor</th><th>Ratio</th><th>Compression</th><th>Decompression</th> + </tr> + <tr> + <th>LZ4 (r101)</th><th>2.084</th><th>422 MB/s</th><th>1820 MB/s</th> + </tr> + <tr> + <th>LZO 2.06</th><th>2.106</th><th>414 MB/s</th><th>600 MB/s</th> + </tr> + <tr> + <th>QuickLZ 1.5.1b6</th><th>2.237</th><th>373 MB/s</th><th>420 MB/s</th> + </tr> + <tr> + <th>Snappy 1.1.0</th><th>2.091</th><th>323 MB/s</th><th>1070 MB/s</th> + </tr> + <tr> + <th>LZF</th><th>2.077</th><th>270 MB/s</th><th>570 MB/s</th> + </tr> + <tr> + <th>zlib 1.2.8 -1</th><th>2.730</th><th>65 MB/s</th><th>280 MB/s</th> + </tr> + <tr> + <th>LZ4 HC (r101)</th><th>2.720</th><th>25 MB/s</th><th>2080 MB/s</th> + </tr> + <tr> + <th>zlib 1.2.8 -6</th><th>3.099</th><th>21 MB/s</th><th>300 MB/s</th> + </tr> +</table> + @@ -1,7 +1,7 @@ /* LZ4 - Fast LZ compression algorithm Header File - Copyright (C) 2011-2013, Yann Collet. + Copyright (C) 2011-2014, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -1,7 +1,7 @@ /* LZ4 HC - High Compression Mode of LZ4 Header File - Copyright (C) 2011-2013, Yann Collet. + Copyright (C) 2011-2014, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without diff --git a/programs/Makefile b/programs/Makefile index fc42ac0..8ec0b85 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -41,6 +41,7 @@ BINDIR=$(PREFIX)/bin MANDIR=$(PREFIX)/share/man/man1 LZ4DIR=.. +TEST_FILES = COPYING # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) @@ -85,11 +86,15 @@ clean: ifneq (,$(filter $(shell uname),Linux Darwin)) install: lz4 lz4c + @echo Installing binaries @install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ @install -m 755 lz4 $(DESTDIR)$(BINDIR)/lz4 @ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4cat @install -m 755 lz4c $(DESTDIR)$(BINDIR)/lz4c + @echo Installing man pages @install -m 644 lz4.1 $(DESTDIR)$(MANDIR)/lz4.1 + @install -m 644 lz4c.1 $(DESTDIR)$(MANDIR)/lz4c.1 + @install -m 644 lz4cat.1 $(DESTDIR)$(MANDIR)/lz4cat.1 @echo lz4 installation completed uninstall: @@ -99,4 +104,24 @@ uninstall: [ -f $(DESTDIR)$(MANDIR)/lz4.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4.1 @echo lz4 successfully uninstalled +test: test-lz4 test-lz4c test-lz4c32 test-fuzzer test-fuzzer32 test-fullbench test-fullbench32 + +test-lz4: + +test-lz4c: + +test-lz4c32: + +test-fuzzer: fuzzer + ./fuzzer --no-prompt + +test-fuzzer32: fuzzer32 + ./fuzzer32 --no-prompt + +test-fullbench: fullbench + ./fullbench --no-prompt $(TEST_FILES) + +test-fullbench32: fullbench32 + ./fullbench32 --no-prompt $(TEST_FILES) + endif diff --git a/programs/bench.c b/programs/bench.c index 2eabf61..8d68098 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -1,6 +1,6 @@ /* bench.c - Demo program to benchmark open-source compression algorithm - Copyright (C) Yann Collet 2012-2013 + Copyright (C) Yann Collet 2012-2014 GPL v2 License This program is free software; you can redistribute it and/or modify diff --git a/programs/bench.h b/programs/bench.h index ed801d4..a0b7ed2 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -1,6 +1,6 @@ /* bench.h - Demo program to benchmark open-source compression algorithm - Copyright (C) Yann Collet 2012-2013 + Copyright (C) Yann Collet 2012-2014 This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/programs/fullbench.c b/programs/fullbench.c index 8e87019..6304029 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -1,6 +1,6 @@ /* bench.c - Demo program to benchmark open-source compression algorithm - Copyright (C) Yann Collet 2012-2013 + Copyright (C) Yann Collet 2012-2014 GPL v2 License This program is free software; you can redistribute it and/or modify @@ -50,6 +50,7 @@ #include <stdio.h> // fprintf, fopen, ftello64 #include <sys/types.h> // stat64 #include <sys/stat.h> // stat64 +#include <string.h> // strcmp // Use ftime() if gettimeofday() is not available on your target #if defined(BMK_LEGACY_TIMER) @@ -138,6 +139,7 @@ struct chunkParameters // MACRO //************************************** #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define PROGRESS(...) no_prompt ? 0 : DISPLAY(__VA_ARGS__) @@ -151,6 +153,7 @@ static int compressionTest = 1; static int decompressionTest = 1; static int compressionAlgo = ALL_COMPRESSORS; static int decompressionAlgo = ALL_DECOMPRESSORS; +static int no_prompt = 0; void BMK_SetBlocksize(int bsize) { @@ -281,12 +284,12 @@ static inline int local_LZ4_compress_limitedOutput_continue(const char* in, char static void* stateLZ4HC; static inline int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize) { - return LZ4_compress_withState(stateLZ4HC, in, out, inSize); + return LZ4_compressHC_withStateHC(stateLZ4HC, in, out, inSize); } static inline int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize) { - return LZ4_compress_limitedOutput_withState(stateLZ4HC, in, out, inSize, LZ4_compressBound(inSize)); + return LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, in, out, inSize, LZ4_compressBound(inSize)); } static inline int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize) @@ -478,7 +481,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) double averageTime; int milliTime; - DISPLAY("%1i-%-19.19s : %9i ->\r", loopNb, cName, (int)benchedSize); + PROGRESS("%1i-%-19.19s : %9i ->\r", loopNb, cName, (int)benchedSize); { size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } // warmimg up memory nb_loops = 0; @@ -502,7 +505,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) if (averageTime < bestTime) bestTime = averageTime; cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize; ratio = (double)cSize/(double)benchedSize*100.; - DISPLAY("%1i-%-19.19s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); + PROGRESS("%1i-%-19.19s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); } if (ratio<100.) @@ -547,7 +550,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) int milliTime; U32 crcDecoded; - DISPLAY("%1i-%-24.24s :%10i ->\r", loopNb, dName, (int)benchedSize); + PROGRESS("%1i-%-24.24s :%10i ->\r", loopNb, dName, (int)benchedSize); nb_loops = 0; milliTime = BMK_GetMilliStart(); @@ -567,7 +570,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) averageTime = (double)milliTime / nb_loops; if (averageTime < bestTime) bestTime = averageTime; - DISPLAY("%1i-%-24.24s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.); + PROGRESS("%1i-%-24.24s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.); // CRC Checking crcDecoded = XXH32(orig_buff, (int)benchedSize, 0); @@ -627,7 +630,7 @@ int usage_advanced() { DISPLAY( "\nAdvanced options :\n"); DISPLAY( " -c# : test only compression function # [%c-%c]\n", MINCOMPRESSIONCHAR, MAXCOMPRESSIONCHAR); - DISPLAY( " -d# : test only compression function # [%c-%c]\n", MINDECOMPRESSIONCHAR, MAXDECOMPRESSIONCHAR); + DISPLAY( " -d# : test only decompression function # [%c-%c]\n", MINDECOMPRESSIONCHAR, MAXDECOMPRESSIONCHAR); DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS); DISPLAY( " -B# : Block size [4-7](default : 7)\n"); //DISPLAY( " -BD : Block dependency (improve compression ratio)\n"); @@ -658,6 +661,11 @@ int main(int argc, char** argv) char* argument = argv[i]; if(!argument) continue; // Protection if argument empty + if (!strcmp(argument, "--no-prompt")) + { + no_prompt = 1; + continue; + } // Decode command (note : aggregated commands are allowed) if (argument[0]=='-') diff --git a/programs/fuzzer.c b/programs/fuzzer.c index a2bd33a..ba4f960 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -34,6 +34,7 @@ #include <stdlib.h> #include <stdio.h> // fgets, sscanf #include <sys/timeb.h> // timeb +#include <string.h> // strcmp #include "lz4.h" #include "lz4hc.h" @@ -121,8 +122,8 @@ int FUZ_SecurityTest() } -//int main(int argc, char *argv[]) { -int main() { +int main(int argc, char** argv) { + const int no_prompt = (argc > 1) && (!strcmp(argv[1], "--no-prompt")); unsigned long long bytes = 0; unsigned long long cbytes = 0; unsigned long long hcbytes = 0; @@ -136,14 +137,14 @@ int main() { int i, j, k, ret, len, lenHC, attemptNb; char userInput[30] = {0}; # define FUZ_CHECKTEST(cond, message) if (cond) { printf("Test %i : %s : seed %u, cycle %i \n", testNb, message, seed, attemptNb); goto _output_error; } -# define FUZ_DISPLAYTEST testNb++; printf("%2i\b\b", testNb); +# define FUZ_DISPLAYTEST testNb++; no_prompt ? 0 : printf("%2i\b\b", testNb); void* stateLZ4 = malloc(LZ4_sizeofState()); void* stateLZ4HC = malloc(LZ4_sizeofStateHC()); printf("starting LZ4 fuzzer (%s)\n", LZ4_VERSION); printf("Select an Initialisation number (default : random) : "); fflush(stdout); - if ( fgets(userInput, sizeof userInput, stdin) ) + if ( no_prompt || fgets(userInput, sizeof userInput, stdin) ) { if ( sscanf(userInput, "%u", &seed) == 1 ) {} else seed = FUZ_GetMilliSpan(timestamp); @@ -160,7 +161,17 @@ int main() { { int testNb = 0; - printf("\r%7i /%7i - ", attemptNb, NB_ATTEMPTS); + // note : promptThrottle is throtting stdout to prevent + // Travis-CI's output limit (10MB) and false hangup detection. + const int promptThrottle = (attemptNb % (NB_ATTEMPTS / 100) == 0); + if (!no_prompt || attemptNb == 0 || promptThrottle) + { + printf("\r%7i /%7i - ", attemptNb, NB_ATTEMPTS); + if (no_prompt) + { + fflush(stdout); + } + } for (j = 0; j < NUM_SEQ; j++) { seeds[j] = FUZ_rand(&randState) << 8; @@ -293,10 +304,10 @@ int main() { printf("all tests completed successfully \n"); printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100); printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100); - getchar(); + if(!no_prompt) getchar(); return 0; _output_error: - getchar(); + if(!no_prompt) getchar(); return 1; } diff --git a/programs/lz4c.1 b/programs/lz4c.1 new file mode 100644 index 0000000..fed6c8b --- /dev/null +++ b/programs/lz4c.1 @@ -0,0 +1,33 @@ +\" +\" lz4c.1: This is a manual page for 'lz4c' program. This file is part of the +\" lz4 <https://code.google.com/p/lz4/> project. +\" + +\" No hyphenation +.hy 0 +.nr HY 0 + +.TH lz4c "1" "2014-04-15" "lz4c" "User Commands" +.SH NAME +\fBlz4\fR - Extremely fast compression algorithm + +.SH SYNOPSIS +.TP 5 +\fBlz4c\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE> + +.SH DESCRIPTION +.PP +\fBlz4c\fR is the legacy version of \fBlz4\fR. +As such, it supports older supplementary legacy commands. +\fBlz4c\fR is now deprecated. +It is recommended to use \fBlz4\fR instead whenever possible. + +To get a list of commands specific to lz4c, do : +lz4c -h + + +.SH BUGS +Report bugs at:- https://code.google.com/p/lz4/ + +.SH AUTHOR +Yann Collet
\ No newline at end of file diff --git a/programs/lz4cat.1 b/programs/lz4cat.1 new file mode 100644 index 0000000..fb33fd5 --- /dev/null +++ b/programs/lz4cat.1 @@ -0,0 +1,34 @@ +\" +\" lz4cat.1: This is a manual page for 'lz4cat' program. This file is part of +\" the lz4 <https://code.google.com/p/lz4/> project. +\" + +\" No hyphenation +.hy 0 +.nr HY 0 + +.TH lz4cat "1" "2014-04-15" "lz4cat" "User Commands" +.SH NAME +\fBlz4cat\fR - Extremely fast compression algorithm + +.SH SYNOPSIS +.TP 5 +\fBlz4cat\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE> + +.SH DESCRIPTION +.PP +\fBlz4\fR is an extremely fast lossless compression algorithm. + +\fBlz4cat\fR is an utility based on \fBlz4\fR. + +\fBlz4cat\fR is equivalent to \fBlz4 -cd\fR, +which forces decompression and redirect its output to the console. + +All other options are the same as \fBlz4\fR ones (man lz4). + + +.SH BUGS +Report bugs at:- https://code.google.com/p/lz4/ + +.SH AUTHOR +Yann Collet
\ No newline at end of file diff --git a/programs/lz4cli.c b/programs/lz4cli.c index fc5bd81..934c2bb 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -1,6 +1,6 @@ /* LZ4cli.c - LZ4 Command Line Interface - Copyright (C) Yann Collet 2011-2013 + Copyright (C) Yann Collet 2011-2014 GPL v2 License This program is free software; you can redistribute it and/or modify @@ -407,11 +407,11 @@ int main(int argc, char** argv) int B = argument[1] - '0'; int S = 1 << (8 + 2*B); BMK_SetBlocksize(S); - LZ4IO_setBlockSizeID(B); + blockSize = LZ4IO_setBlockSizeID(B); argument++; break; } - case 'D': LZ4IO_setBlockMode(independentBlocks); argument++; break; + case 'D': LZ4IO_setBlockMode(chainedBlocks); argument++; break; case 'X': LZ4IO_setBlockChecksumMode(1); argument ++; break; default : exitBlockProperties=1; } diff --git a/programs/lz4io.c b/programs/lz4io.c index ae8b776..05955cf 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -1,6 +1,6 @@ /* LZ4io.c - LZ4 File/Stream Interface - Copyright (C) Yann Collet 2011-2013 + Copyright (C) Yann Collet 2011-2014 GPL v2 License This program is free software; you can redistribute it and/or modify diff --git a/programs/lz4io.h b/programs/lz4io.h index cb35ec0..9c3b217 100644 --- a/programs/lz4io.h +++ b/programs/lz4io.h @@ -1,6 +1,6 @@ /* LZ4io.h - LZ4 File/Stream Interface - Copyright (C) Yann Collet 2011-2013 + Copyright (C) Yann Collet 2011-2014 GPL v2 License This program is free software; you can redistribute it and/or modify |