Merge pull request #3 from Cyan4973/betar117

Beta

17 files changed, 204 insertions, 28 deletions

diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..695c19f
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,6 @@
+language: c
+compiler: gcc
+script: make test
+before_install:
+  - sudo apt-get update -qq
+  - sudo apt-get install -qq gcc-multilib
diff --git a/LICENSE b/LICENSE
index 04403e4..b566df3 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,5 +1,5 @@
 LZ4 Library
-Copyright (c) 2013, Yann Collet
+Copyright (c) 2011-2014, Yann Collet
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification,
diff --git a/Makefile b/Makefile
index 4ceacb9..e5797c7 100644
--- a/Makefile
+++ b/Makefile
@@ -30,7 +30,7 @@
 #  - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c
 # ################################################################
 
-export RELEASE=r116
+export RELEASE=r117
 LIBVER_MAJOR=1
 LIBVER_MINOR=0
 LIBVER_PATCH=0
@@ -69,13 +69,14 @@ else
 endif
 
 TEXT = lz4.c lz4.h lz4hc.c lz4hc.h \
-	lz4_format_description.txt Makefile NEWS LICENSE \
+	lz4_format_description.txt Makefile NEWS LICENSE README.md \
 	cmake_unofficial/CMakeLists.txt \
 	$(PRGDIR)/fullbench.c $(PRGDIR)/fuzzer.c $(PRGDIR)/lz4cli.c \
 	$(PRGDIR)/lz4io.c $(PRGDIR)/lz4io.h \
 	$(PRGDIR)/bench.c $(PRGDIR)/bench.h \
 	$(PRGDIR)/xxhash.c $(PRGDIR)/xxhash.h \
-	$(PRGDIR)/lz4.1 $(PRGDIR)/Makefile $(PRGDIR)/COPYING
+	$(PRGDIR)/lz4.1 $(PRGDIR)/lz4c.1 $(PRGDIR)/lz4cat.1 \
+	$(PRGDIR)/Makefile $(PRGDIR)/COPYING
 NONTEXT = LZ4_Streaming_Format.odt
 SOURCES = $(TEXT) $(NONTEXT)
 
@@ -144,4 +145,7 @@ dist: clean
 	@sha1sum $(DISTRIBNAME) > $(DISTRIBNAME).sha1
 	@echo Distribution $(DISTRIBNAME) built
 
+test: lz4programs
+	@cd $(PRGDIR); $(MAKE) -e $@
+
 endif
diff --git a/NEWS b/NEWS
index 39a19f9..d850cda 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,9 @@
+r117:
+Added : man pages for lz4c and lz4cat
+Added : automated tests on Travis, thanks to Takayuki Matsuoka !
+fix : block-dependency command line (issue 127)
+fix : lz4fullbench (issue 128)
+
 r116:
 hotfix (issue 124 & 125)
 
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..98ba7c6
--- /dev/null
+++ b/README.md
@@ -0,0 +1,49 @@
+LZ4 - Extremely fast compression
+================================
+
+LZ4 is lossless compression algorithm, providing compression speed at 400 MB/s per core, scalable with multi-cores CPU. It also features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limits on multi-core systems.
+A high compression derivative, called LZ4_HC, is also provided. It trades CPU time for compression ratio.
+
+This is an official mirror of LZ4 project, [hosted on Google Code](http://code.google.com/p/lz4/).
+The intention is to offer github's capabilities to lz4 users, such as cloning, branch, or source download.
+
+The "master" branch will reflect, the status of lz4 at its official homepage. Other branches will also exist, typically to fix some open issues or new requirements, and be available for testing before merge into master.
+
+
+Benchmarks
+-------------------------
+
+The benchmark uses the [Open-Source Benchmark program by m^2 (v0.14.2)](http://encode.ru/threads/1371-Filesystem-benchmark?p=33548&viewfull=1#post33548) compiled with GCC v4.6.1 on Linux Ubuntu 64-bits v11.10,
+The reference system uses a Core i5-3340M @2.7GHz.
+Benchmark evaluates the compression of reference [Silesia Corpus](http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia) in single-thread mode.
+
+<table>
+  <tr>
+    <th>Compressor</th><th>Ratio</th><th>Compression</th><th>Decompression</th>
+  </tr>
+  <tr>
+    <th>LZ4 (r101)</th><th>2.084</th><th>422 MB/s</th><th>1820 MB/s</th>
+  </tr>
+  <tr>
+    <th>LZO 2.06</th><th>2.106</th><th>414 MB/s</th><th>600 MB/s</th>
+  </tr>
+  <tr>
+    <th>QuickLZ 1.5.1b6</th><th>2.237</th><th>373 MB/s</th><th>420 MB/s</th>
+  </tr>
+  <tr>
+    <th>Snappy 1.1.0</th><th>2.091</th><th>323 MB/s</th><th>1070 MB/s</th>
+  </tr>
+  <tr>
+    <th>LZF</th><th>2.077</th><th>270 MB/s</th><th>570 MB/s</th>
+  </tr>
+  <tr>
+    <th>zlib 1.2.8 -1</th><th>2.730</th><th>65 MB/s</th><th>280 MB/s</th>
+  </tr>
+  <tr>
+    <th>LZ4 HC (r101)</th><th>2.720</th><th>25 MB/s</th><th>2080 MB/s</th>
+  </tr>
+  <tr>
+    <th>zlib 1.2.8 -6</th><th>3.099</th><th>21 MB/s</th><th>300 MB/s</th>
+  </tr>
+</table>
+
diff --git a/lz4.h b/lz4.h
index 4b8a15b..3bf0a48 100644
--- a/lz4.h
+++ b/lz4.h
@@ -1,7 +1,7 @@
 /*
    LZ4 - Fast LZ compression algorithm
    Header File
-   Copyright (C) 2011-2013, Yann Collet.
+   Copyright (C) 2011-2014, Yann Collet.
    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
diff --git a/lz4hc.h b/lz4hc.h
index d3f81a8..b810978 100644
--- a/lz4hc.h
+++ b/lz4hc.h
@@ -1,7 +1,7 @@
 /*
    LZ4 HC - High Compression Mode of LZ4
    Header File
-   Copyright (C) 2011-2013, Yann Collet.
+   Copyright (C) 2011-2014, Yann Collet.
    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
diff --git a/programs/Makefile b/programs/Makefile
index fc42ac0..8ec0b85 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -41,6 +41,7 @@ BINDIR=$(PREFIX)/bin
 MANDIR=$(PREFIX)/share/man/man1
 LZ4DIR=..
 
+TEST_FILES = COPYING
 
 # Define *.exe as extension for Windows systems
 ifneq (,$(filter Windows%,$(OS)))
@@ -85,11 +86,15 @@ clean:
 ifneq (,$(filter $(shell uname),Linux Darwin))
 
 install: lz4 lz4c
+	@echo Installing binaries
 	@install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
 	@install -m 755 lz4   $(DESTDIR)$(BINDIR)/lz4
 	@ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4cat
 	@install -m 755 lz4c  $(DESTDIR)$(BINDIR)/lz4c
+	@echo Installing man pages
 	@install -m 644 lz4.1 $(DESTDIR)$(MANDIR)/lz4.1
+	@install -m 644 lz4c.1 $(DESTDIR)$(MANDIR)/lz4c.1
+	@install -m 644 lz4cat.1 $(DESTDIR)$(MANDIR)/lz4cat.1
 	@echo lz4 installation completed
 
 uninstall:
@@ -99,4 +104,24 @@ uninstall:
 	[ -f $(DESTDIR)$(MANDIR)/lz4.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4.1
 	@echo lz4 successfully uninstalled
 
+test: test-lz4 test-lz4c test-lz4c32 test-fuzzer test-fuzzer32 test-fullbench test-fullbench32
+
+test-lz4:
+
+test-lz4c:
+
+test-lz4c32:
+
+test-fuzzer: fuzzer
+	./fuzzer --no-prompt
+
+test-fuzzer32: fuzzer32
+	./fuzzer32 --no-prompt
+
+test-fullbench: fullbench
+	./fullbench --no-prompt $(TEST_FILES)
+
+test-fullbench32: fullbench32
+	./fullbench32 --no-prompt $(TEST_FILES)
+
 endif
diff --git a/programs/bench.c b/programs/bench.c
index 2eabf61..8d68098 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -1,6 +1,6 @@
 /*
     bench.c - Demo program to benchmark open-source compression algorithm
-    Copyright (C) Yann Collet 2012-2013
+    Copyright (C) Yann Collet 2012-2014
     GPL v2 License
 
     This program is free software; you can redistribute it and/or modify
diff --git a/programs/bench.h b/programs/bench.h
index ed801d4..a0b7ed2 100644
--- a/programs/bench.h
+++ b/programs/bench.h
@@ -1,6 +1,6 @@
 /*
     bench.h - Demo program to benchmark open-source compression algorithm
-    Copyright (C) Yann Collet 2012-2013
+    Copyright (C) Yann Collet 2012-2014
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
diff --git a/programs/fullbench.c b/programs/fullbench.c
index 8e87019..6304029 100644
--- a/programs/fullbench.c
+++ b/programs/fullbench.c
@@ -1,6 +1,6 @@
 /*
     bench.c - Demo program to benchmark open-source compression algorithm
-    Copyright (C) Yann Collet 2012-2013
+    Copyright (C) Yann Collet 2012-2014
     GPL v2 License
 
     This program is free software; you can redistribute it and/or modify
@@ -50,6 +50,7 @@
 #include <stdio.h>       // fprintf, fopen, ftello64
 #include <sys/types.h>   // stat64
 #include <sys/stat.h>    // stat64
+#include <string.h>      // strcmp
 
 // Use ftime() if gettimeofday() is not available on your target
 #if defined(BMK_LEGACY_TIMER)
@@ -138,6 +139,7 @@ struct chunkParameters
 // MACRO
 //**************************************
 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
+#define PROGRESS(...) no_prompt ? 0 : DISPLAY(__VA_ARGS__)
 
 
 
@@ -151,6 +153,7 @@ static int compressionTest = 1;
 static int decompressionTest = 1;
 static int compressionAlgo = ALL_COMPRESSORS;
 static int decompressionAlgo = ALL_DECOMPRESSORS;
+static int no_prompt = 0;
 
 void BMK_SetBlocksize(int bsize)
 {
@@ -281,12 +284,12 @@ static inline int local_LZ4_compress_limitedOutput_continue(const char* in, char
 static void* stateLZ4HC;
 static inline int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize)
 {
-    return LZ4_compress_withState(stateLZ4HC, in, out, inSize);
+    return LZ4_compressHC_withStateHC(stateLZ4HC, in, out, inSize);
 }
 
 static inline int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize)
 {
-    return LZ4_compress_limitedOutput_withState(stateLZ4HC, in, out, inSize, LZ4_compressBound(inSize));
+    return LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, in, out, inSize, LZ4_compressBound(inSize));
 }
 
 static inline int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize)
@@ -478,7 +481,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
                 double averageTime;
                 int milliTime;
 
-                DISPLAY("%1i-%-19.19s : %9i ->\r", loopNb, cName, (int)benchedSize);
+                PROGRESS("%1i-%-19.19s : %9i ->\r", loopNb, cName, (int)benchedSize);
                 { size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; }     // warmimg up memory
 
                 nb_loops = 0;
@@ -502,7 +505,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
                 if (averageTime < bestTime) bestTime = averageTime;
                 cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;
                 ratio = (double)cSize/(double)benchedSize*100.;
-                DISPLAY("%1i-%-19.19s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
+                PROGRESS("%1i-%-19.19s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
             }
 
             if (ratio<100.)
@@ -547,7 +550,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
                 int milliTime;
                 U32 crcDecoded;
 
-                DISPLAY("%1i-%-24.24s :%10i ->\r", loopNb, dName, (int)benchedSize);
+                PROGRESS("%1i-%-24.24s :%10i ->\r", loopNb, dName, (int)benchedSize);
 
                 nb_loops = 0;
                 milliTime = BMK_GetMilliStart();
@@ -567,7 +570,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
                 averageTime = (double)milliTime / nb_loops;
                 if (averageTime < bestTime) bestTime = averageTime;
 
-                DISPLAY("%1i-%-24.24s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.);
+                PROGRESS("%1i-%-24.24s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.);
 
                 // CRC Checking
                 crcDecoded = XXH32(orig_buff, (int)benchedSize, 0);
@@ -627,7 +630,7 @@ int usage_advanced()
 {
     DISPLAY( "\nAdvanced options :\n");
     DISPLAY( " -c#    : test only compression function # [%c-%c]\n", MINCOMPRESSIONCHAR, MAXCOMPRESSIONCHAR);
-    DISPLAY( " -d#    : test only compression function # [%c-%c]\n", MINDECOMPRESSIONCHAR, MAXDECOMPRESSIONCHAR);
+    DISPLAY( " -d#    : test only decompression function # [%c-%c]\n", MINDECOMPRESSIONCHAR, MAXDECOMPRESSIONCHAR);
     DISPLAY( " -i#    : iteration loops [1-9](default : %i)\n", NBLOOPS);
     DISPLAY( " -B#    : Block size [4-7](default : 7)\n");
     //DISPLAY( " -BD    : Block dependency (improve compression ratio)\n");
@@ -658,6 +661,11 @@ int main(int argc, char** argv)
         char* argument = argv[i];
 
         if(!argument) continue;   // Protection if argument empty
+        if (!strcmp(argument, "--no-prompt"))
+        {
+            no_prompt = 1;
+            continue;
+        }
 
         // Decode command (note : aggregated commands are allowed)
         if (argument[0]=='-')
diff --git a/programs/fuzzer.c b/programs/fuzzer.c
index a2bd33a..ba4f960 100644
--- a/programs/fuzzer.c
+++ b/programs/fuzzer.c
@@ -34,6 +34,7 @@
 #include <stdlib.h>
 #include <stdio.h>      // fgets, sscanf
 #include <sys/timeb.h>  // timeb
+#include <string.h>     // strcmp
 #include "lz4.h"
 #include "lz4hc.h"
 
@@ -121,8 +122,8 @@ int FUZ_SecurityTest()
 }
 
 
-//int main(int argc, char *argv[]) {
-int main() {
+int main(int argc, char** argv) {
+        const int no_prompt = (argc > 1) && (!strcmp(argv[1], "--no-prompt"));
         unsigned long long bytes = 0;
         unsigned long long cbytes = 0;
         unsigned long long hcbytes = 0;
@@ -136,14 +137,14 @@ int main() {
         int i, j, k, ret, len, lenHC, attemptNb;
         char userInput[30] = {0};
 #       define FUZ_CHECKTEST(cond, message) if (cond) { printf("Test %i : %s : seed %u, cycle %i \n", testNb, message, seed, attemptNb); goto _output_error; }
-#       define FUZ_DISPLAYTEST              testNb++; printf("%2i\b\b", testNb);
+#       define FUZ_DISPLAYTEST              testNb++; no_prompt ? 0 : printf("%2i\b\b", testNb);
         void* stateLZ4   = malloc(LZ4_sizeofState());
         void* stateLZ4HC = malloc(LZ4_sizeofStateHC());
 
         printf("starting LZ4 fuzzer (%s)\n", LZ4_VERSION);
         printf("Select an Initialisation number (default : random) : ");
         fflush(stdout);
-        if ( fgets(userInput, sizeof userInput, stdin) )
+        if ( no_prompt || fgets(userInput, sizeof userInput, stdin) )
         {
             if ( sscanf(userInput, "%u", &seed) == 1 ) {}
             else seed = FUZ_GetMilliSpan(timestamp);
@@ -160,7 +161,17 @@ int main() {
         {
             int testNb = 0;
 
-            printf("\r%7i /%7i   - ", attemptNb, NB_ATTEMPTS);
+            // note : promptThrottle is throtting stdout to prevent
+            //        Travis-CI's output limit (10MB) and false hangup detection.
+            const int promptThrottle = (attemptNb % (NB_ATTEMPTS / 100) == 0);
+            if (!no_prompt || attemptNb == 0 || promptThrottle)
+            {
+                printf("\r%7i /%7i   - ", attemptNb, NB_ATTEMPTS);
+                if (no_prompt)
+                {
+                    fflush(stdout);
+                }
+            }
 
             for (j = 0; j < NUM_SEQ; j++) {
                     seeds[j] = FUZ_rand(&randState) << 8;
@@ -293,10 +304,10 @@ int main() {
         printf("all tests completed successfully \n");
         printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100);
         printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100);
-        getchar();
+        if(!no_prompt) getchar();
         return 0;
 
 _output_error:
-        getchar();
+        if(!no_prompt) getchar();
         return 1;
 }
diff --git a/programs/lz4c.1 b/programs/lz4c.1
new file mode 100644
index 0000000..fed6c8b
--- /dev/null
+++ b/programs/lz4c.1
@@ -0,0 +1,33 @@
+\"
+\" lz4c.1: This is a manual page for 'lz4c' program. This file is part of the
+\" lz4 <https://code.google.com/p/lz4/> project.
+\"
+
+\" No hyphenation
+.hy 0
+.nr HY 0
+
+.TH lz4c "1" "2014-04-15" "lz4c" "User Commands"
+.SH NAME
+\fBlz4\fR - Extremely fast compression algorithm
+
+.SH SYNOPSIS
+.TP 5
+\fBlz4c\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE>
+
+.SH DESCRIPTION
+.PP
+\fBlz4c\fR is the legacy version of \fBlz4\fR.
+As such, it supports older supplementary legacy commands.
+\fBlz4c\fR is now deprecated.
+It is recommended to use \fBlz4\fR instead whenever possible.
+
+To get a list of commands specific to lz4c, do :
+lz4c -h
+
+
+.SH BUGS
+Report bugs at:- https://code.google.com/p/lz4/
+
+.SH AUTHOR
+Yann Collet
\ No newline at end of file
diff --git a/programs/lz4cat.1 b/programs/lz4cat.1
new file mode 100644
index 0000000..fb33fd5
--- /dev/null
+++ b/programs/lz4cat.1
@@ -0,0 +1,34 @@
+\"
+\" lz4cat.1: This is a manual page for 'lz4cat' program. This file is part of
+\" the lz4 <https://code.google.com/p/lz4/> project.
+\"
+
+\" No hyphenation
+.hy 0
+.nr HY 0
+
+.TH lz4cat "1" "2014-04-15" "lz4cat" "User Commands"
+.SH NAME
+\fBlz4cat\fR - Extremely fast compression algorithm
+
+.SH SYNOPSIS
+.TP 5
+\fBlz4cat\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE>
+
+.SH DESCRIPTION
+.PP
+\fBlz4\fR is an extremely fast lossless compression algorithm.
+
+\fBlz4cat\fR is an utility based on \fBlz4\fR.
+
+\fBlz4cat\fR is equivalent to \fBlz4 -cd\fR,
+which forces decompression and redirect its output to the console.
+
+All other options are the same as \fBlz4\fR ones (man lz4).
+
+
+.SH BUGS
+Report bugs at:- https://code.google.com/p/lz4/
+
+.SH AUTHOR
+Yann Collet
\ No newline at end of file
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index fc5bd81..934c2bb 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -1,6 +1,6 @@
 /*
   LZ4cli.c - LZ4 Command Line Interface
-  Copyright (C) Yann Collet 2011-2013
+  Copyright (C) Yann Collet 2011-2014
   GPL v2 License
 
   This program is free software; you can redistribute it and/or modify
@@ -407,11 +407,11 @@ int main(int argc, char** argv)
                             int B = argument[1] - '0';
                             int S = 1 << (8 + 2*B);
                             BMK_SetBlocksize(S);
-                            LZ4IO_setBlockSizeID(B);
+                            blockSize = LZ4IO_setBlockSizeID(B);
                             argument++;
                             break;
                         }
-                        case 'D': LZ4IO_setBlockMode(independentBlocks); argument++; break;
+                        case 'D': LZ4IO_setBlockMode(chainedBlocks); argument++; break;
                         case 'X': LZ4IO_setBlockChecksumMode(1); argument ++; break;
                         default : exitBlockProperties=1;
                         }
diff --git a/programs/lz4io.c b/programs/lz4io.c
index ae8b776..05955cf 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -1,6 +1,6 @@
 /*
   LZ4io.c - LZ4 File/Stream Interface
-  Copyright (C) Yann Collet 2011-2013
+  Copyright (C) Yann Collet 2011-2014
   GPL v2 License
 
   This program is free software; you can redistribute it and/or modify
diff --git a/programs/lz4io.h b/programs/lz4io.h
index cb35ec0..9c3b217 100644
--- a/programs/lz4io.h
+++ b/programs/lz4io.h
@@ -1,6 +1,6 @@
 /*
   LZ4io.h - LZ4 File/Stream Interface
-  Copyright (C) Yann Collet 2011-2013
+  Copyright (C) Yann Collet 2011-2014
   GPL v2 License
 
   This program is free software; you can redistribute it and/or modify