Merge pull request #111 from Cyan4973/devr130lz4-r130

Dev

13 files changed, 285 insertions, 42 deletions

diff --git a/.travis.yml b/.travis.yml
index 398a0f1..6c505bc 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,6 +8,7 @@ before_install:
   - sudo apt-get install -qq g++-multilib
   - sudo apt-get install -qq gcc-multilib
   - sudo apt-get install -qq valgrind
+  - sudo apt-get install -qq python3
 
 env:
   - LZ4_TRAVIS_CI_ENV=travis-install
@@ -18,6 +19,7 @@ env:
   - LZ4_TRAVIS_CI_ENV=staticAnalyze
   - LZ4_TRAVIS_CI_ENV=gpptest
   - LZ4_TRAVIS_CI_ENV=armtest
+  - LZ4_TRAVIS_CI_ENV=versionstest
   - LZ4_TRAVIS_CI_ENV=test-lz4
   - LZ4_TRAVIS_CI_ENV=test-lz4c
   - LZ4_TRAVIS_CI_ENV=test-lz4c32
diff --git a/Makefile b/Makefile
index 88c4016..ac48b65 100644
--- a/Makefile
+++ b/Makefile
@@ -31,7 +31,7 @@
 # ################################################################
 
 # Version number
-export VERSION=129
+export VERSION=130
 export RELEASE=r$(VERSION)
 
 DESTDIR?=
@@ -71,6 +71,7 @@ clean:
 	@cd $(PRGDIR); $(MAKE) clean > $(VOID)
 	@cd $(LZ4DIR); $(MAKE) clean > $(VOID)
 	@cd examples;  $(MAKE) clean > $(VOID)
+	@cd test;      $(MAKE) clean > $(VOID)
 	@echo Cleaning completed
 
 
@@ -113,6 +114,9 @@ armtest: clean
 	cd lib; $(MAKE) -e all CC=arm-linux-gnueabi-gcc CPPFLAGS="-Werror"
 	cd programs; $(MAKE) -e bins CC=arm-linux-gnueabi-gcc CPPFLAGS="-Werror"
 
+versionstest: clean
+	@cd test; $(MAKE)
+
 streaming-examples:
 	cd examples; $(MAKE) -e test
 
diff --git a/NEWS b/NEWS
index 08622f2..8347f63 100644
--- a/NEWS
+++ b/NEWS
@@ -1,5 +1,13 @@
+r130:
+Fixed  : incompatibility sparse mode vs console, reported by Yongwoon Cho (#105)
+Fixed  : LZ4IO exits too early when frame crc not present, reported by Yongwoon Cho (#106)
+Fixed  : incompatibility sparse mode vs append mode, reported by Takayuki Matsuoka (#110)
+Performance fix : big compression speed boost for clang (+30%)
+New    : cross-version test, by Takayuki Matsuoka
+
 r129:
-Added  : LZ4_compress_fast()
+Added  : LZ4_compress_fast(), LZ4_compress_fast_continue()
+Added  : LZ4_compress_destSize()
 Changed: New lz4 and lz4hc compression API. Previous function prototypes still supported.
 Changed: Sparse file support enabled by default
 New    : LZ4 CLI improved performance compressing/decompressing multiple files (#86, kind contribution from Kyle J. Harper & Takayuki Matsuoka)
diff --git a/README.md b/README.md
index d050029..31f6158 100644
--- a/README.md
+++ b/README.md
@@ -49,6 +49,7 @@ in single-thread mode.
 |  LZO 2.06            |  2.108  |  350 MB/s   |    510 MB/s   |
 |  QuickLZ 1.5.1.b6    |  2.238  |  320 MB/s   |    380 MB/s   |
 |  Snappy 1.1.0        |  2.091  |  250 MB/s   |    960 MB/s   |
+|  LZF v3.6            |  2.073  |  175 MB/s   |    500 MB/s   |
 |  zlib 1.2.8 -1       |  2.730  |   59 MB/s   |    250 MB/s   |
 |**LZ4 HC (r129)**     |**2.720**|   22 MB/s   | **1830 MB/s** |
 |  zlib 1.2.8 -6       |  3.099  |   18 MB/s   |    270 MB/s   |
@@ -77,4 +78,4 @@ A list of these sources is maintained on the [LZ4 Homepage].
 [Silesia Corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
 [lz4_Block_format]: lz4_Block_format.md
 [lz4_Frame_format]: lz4_Frame_format.md
-[LZ4 Homepage]: http://www.lz4.info
+[LZ4 Homepage]: http://www.lz4.org
diff --git a/lib/lz4.c b/lib/lz4.c
index 55f2359..08cf6b5 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -45,7 +45,7 @@
 
 /*
  * ACCELERATION_DEFAULT :
- * Select the value of "acceleration" for LZ4_compress_fast() when parameter == 0
+ * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0
  */
 #define ACCELERATION_DEFAULT 1
 
@@ -432,7 +432,7 @@ static const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t t
     return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
 }
 
-static int LZ4_compress_generic(
+FORCE_INLINE int LZ4_compress_generic(
                  void* const ctx,
                  const char* const source,
                  char* const dest,
diff --git a/lib/lz4.h b/lib/lz4.h
index 20e3d48..99c6ebb 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -122,8 +122,8 @@ LZ4_compress_fast() :
     Same as LZ4_compress_default(), but allows to select an "acceleration" factor.
     The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
     It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.
-    An acceleration value of "0" means "use Default value" (see lz4.c)
     An acceleration value of "1" is the same as regular LZ4_compress_default()
+    Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1.
 */
 int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration);
 
@@ -145,7 +145,6 @@ LZ4_compress_destSize() :
     into already allocated buffer 'dest' of size 'targetDestSize'.
     This function either compresses the entire 'source' content into 'dest' if it's large enough,
     or fill 'dest' buffer completely with as much data as possible from 'source'.
-    Original idea by WiredTiger team.
         *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'.
                          New value is necessarily <= old value.
         return : Nb bytes written into 'dest' (necessarily <= targetDestSize)
diff --git a/lz4_Block_format.md b/lz4_Block_format.md
index a120a0b..ea568d8 100644
--- a/lz4_Block_format.md
+++ b/lz4_Block_format.md
@@ -1,25 +1,26 @@
 LZ4 Block Format Description
 ============================
-Last revised: 2015-03-26.
+Last revised: 2015-05-07.
 Author : Yann Collet
 
 
-This small specification intents to provide enough information
-to anyone willing to produce LZ4-compatible compressed data blocks
+This specification is intended for developers
+willing to produce LZ4-compatible compressed data blocks
 using any programming language.
 
 LZ4 is an LZ77-type compressor with a fixed, byte-oriented encoding.
-The most important design principle behind LZ4 is simplicity.
-It helps to create an easy to read and maintain source code.
-It also helps later on for optimizations, compactness, and speed.
 There is no entropy encoder back-end nor framing layer.
-The latter is assumed to be handled by other parts of the system.
+The latter is assumed to be handled by other parts of the system (see [LZ4 Frame format]).
+This design is assumed to favor simplicity and speed.
+It helps later on for optimizations, compactness, and features.
 
-This document only describes the block format,
-not how the LZ4 compressor nor decompressor actually work.
+This document describes only the block format,
+not how the compressor nor decompressor actually work.
 The correctness of the decompressor should not depend
 on implementation details of the compressor, and vice versa.
 
+[LZ4 Frame format]: LZ4_Frame_format.md
+
 
 
 Compressed block format
diff --git a/lz4_Frame_format.md b/lz4_Frame_format.md
index cb41d47..73d3cba 100644
--- a/lz4_Frame_format.md
+++ b/lz4_Frame_format.md
@@ -205,13 +205,13 @@ This is the original (uncompressed) size.
 This information is optional, and only present if the associated flag is set.
 Content size is provided using unsigned 8 Bytes, for a maximum of 16 HexaBytes.
 Format is Little endian.
-This value is informational.
-It can be safely skipped by a conformant decoder.
+This value is informational, typically for display or memory allocation.
+It can be skipped by a decoder, or used to validate content correctness.
 
 __Header Checksum__
 
 One-byte checksum of combined descriptor fields, including optional ones.
-The value is the second byte of xxh32() : ` (xxh32()>>8) & 0xFF } `
+The value is the second byte of xxh32() : ` (xxh32()>>8) & 0xFF `
 using zero as a seed,
 and the full Frame Descriptor as an input
 (including optional fields when they are present).
diff --git a/programs/Makefile b/programs/Makefile
index 39335db..43f1789 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -34,7 +34,7 @@
 # datagen : generates synthetic data samples for tests & benchmarks
 # ##########################################################################
 
-RELEASE?= r129
+RELEASE?= r130
 
 DESTDIR?=
 PREFIX ?= /usr/local
@@ -150,7 +150,7 @@ test-all: test test32
 test-travis: $(TRAVIS_TARGET)
 
 test-lz4-sparse: lz4 datagen
-	@echo ---- test sparse file support ----
+	@echo "\n ---- test sparse file support ----"
 	./datagen -g5M  -P100 > tmpSrc
 	./lz4 -B4D tmpSrc | ./lz4 -dv --sparse > tmpB4
 	diff -s tmpSrc tmpB4
@@ -167,9 +167,25 @@ test-lz4-sparse: lz4 datagen
 	./datagen -s1 -g1200007 -P100 | diff -s - tmpOdd
 	ls -ls tmpOdd
 	@rm tmp*
+	@echo "\n Compatibility with Console :"
+	echo "Hello World 1 !" | ./lz4 | ./lz4 -d -c
+	echo "Hello World 2 !" | ./lz4 | ./lz4 -d | cat
+	echo "Hello World 3 !" | ./lz4 --no-frame-crc | ./lz4 -d -c
+	@echo "\n Compatibility with Append :"
+	./datagen -P100 -g1M > tmp1M
+	cat tmp1M > tmp2M
+	cat tmp1M >> tmp2M
+	./lz4 -B5 -v tmp1M tmpC
+	./lz4 -d -v tmpC tmpR
+	./lz4 -d -v tmpC >> tmpR
+	ls -ls tmp*
+	diff tmp2M tmpR
+	@rm tmp*
+
+
 
 test-lz4-contentSize: lz4 datagen
-	@echo ---- test original size support ----
+	@echo "\n ---- test original size support ----"
 	./datagen -g15M > tmp
 	./lz4 -v tmp | ./lz4 -t
 	./lz4 -v --content-size tmp | ./lz4 -d > tmp2
@@ -182,7 +198,7 @@ test-lz4-contentSize: lz4 datagen
 	@rm tmp*
 
 test-lz4-frame-concatenation: lz4 datagen
-	@echo ---- test frame concatenation ----
+	@echo "\n ---- test frame concatenation ----"
 	@echo -n > empty.test
 	@echo hi > nonempty.test
 	cat nonempty.test empty.test nonempty.test > orig.test
@@ -195,7 +211,7 @@ test-lz4-frame-concatenation: lz4 datagen
 	@echo frame concatenation test completed
 
 test-lz4-multiple: lz4 datagen
-	@echo ---- test multiple files ----
+	@echo "\n ---- test multiple files ----"
 	@./datagen -s1        > tmp1 2> $(VOID)
 	@./datagen -s2 -g100K > tmp2 2> $(VOID)
 	@./datagen -s3 -g1M   > tmp3 2> $(VOID)
@@ -207,10 +223,15 @@ test-lz4-multiple: lz4 datagen
 	./lz4 -f -m tmp1 notHere tmp2; echo $$?
 	@rm tmp*
 
-test-lz4: lz4 datagen test-lz4-multiple test-lz4-sparse test-lz4-contentSize test-lz4-frame-concatenation
-	@echo ---- test lz4 basic compression/decompression ----
+test-lz4-basic: lz4 datagen
+	@echo "\n ---- test lz4 basic compression/decompression ----"
 	./datagen -g0     | ./lz4 -v     | ./lz4 -t
 	./datagen -g16KB  | ./lz4 -9     | ./lz4 -t
+	./datagen -g20KB > tmpSrc
+	./lz4 < tmpSrc | ./lz4 -d > tmpRes
+	diff -q tmpSrc tmpRes
+	./lz4 --no-frame-crc < tmpSrc | ./lz4 -d > tmpRes
+	diff -q tmpSrc tmpRes
 	./datagen         | ./lz4        | ./lz4 -t
 	./datagen -g6M -P99 | ./lz4 -9BD | ./lz4 -t
 	./datagen -g17M   | ./lz4 -9v    | ./lz4 -qt
@@ -218,22 +239,35 @@ test-lz4: lz4 datagen test-lz4-multiple test-lz4-sparse test-lz4-contentSize tes
 	./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -t
 	./datagen -g6GB   | ./lz4 -vqB5D | ./lz4 -qt
 	./datagen -g6GB   | ./lz4 -vq9BD | ./lz4 -qt
-	@echo ---- test pass-through ----
+	@rm tmp*
+
+test-lz4: lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-sparse test-lz4-contentSize test-lz4-frame-concatenation
+	@echo "\n ---- test pass-through ----"
 	./datagen | ./lz4 -tf
 
 test-lz4c: lz4c datagen
+	@echo "\n ---- test lz4c version ----"
 	./datagen -g256MB | ./lz4c -l -v    | ./lz4c   -t
 
-test-lz4c32: lz4 lz4c32 datagen
-	./datagen -g16KB  | ./lz4c32 -9     | ./lz4c32 -t
+test-interop-32-64: lz4 lz4c32 datagen
+	@echo "\n ---- test interoperability 32-bits -vs- 64 bits ----"
 	./datagen -g16KB  | ./lz4c32 -9     | ./lz4    -t
-	./datagen         | ./lz4c32        | ./lz4c32 -t
+	./datagen -P10    | ./lz4    -9B4   | ./lz4c32 -t
 	./datagen         | ./lz4c32        | ./lz4    -t
-	./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4c32 -qt
+	./datagen -g1M    | ./lz4    -3B5   | ./lz4c32 -t
 	./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4    -qt
-	./datagen -g6GB   | ./lz4c32 -vqB5D | ./lz4c32 -qt
+	./datagen -g1G -P90 | ./lz4         | ./lz4c32 -t
 	./datagen -g6GB   | ./lz4c32 -vq9BD | ./lz4    -qt
 
+test-lz4c32-basic: lz4c32 datagen
+	@echo "\n ---- test lz4c32 32-bits version ----"
+	./datagen -g16KB  | ./lz4c32 -9     | ./lz4c32 -t
+	./datagen         | ./lz4c32        | ./lz4c32 -t
+	./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4c32 -qt
+	./datagen -g6GB   | ./lz4c32 -vqB5D | ./lz4c32 -qt
+
+test-lz4c32: test-lz4c32-basic test-interop-32-64
+
 test-fullbench: fullbench
 	./fullbench --no-prompt $(NB_LOOPS) $(TEST_FILES)
 
@@ -253,6 +287,7 @@ test-frametest32: frametest32
 	./frametest32 $(FUZZER_TIME)
 
 test-mem: lz4 datagen fuzzer frametest fullbench
+	@echo "\n ---- valgrind tests : memory analyzer ----"
 	valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID)
 	./datagen -g16KB > tmp
 	valgrind --leak-check=yes --error-exitcode=1 ./lz4 -9 -BD -f tmp $(VOID)
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index 970856d..e5210c9 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -306,7 +306,7 @@ int main(int argc, char** argv)
         if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(0); continue; }
         if (!strcmp(argument, "--content-size")) { LZ4IO_setContentSize(1); continue; }
         if (!strcmp(argument, "--no-content-size")) { LZ4IO_setContentSize(0); continue; }
-        if (!strcmp(argument, "--sparse")) { LZ4IO_setSparseFile(1); continue; }
+        if (!strcmp(argument, "--sparse")) { LZ4IO_setSparseFile(2); continue; }
         if (!strcmp(argument, "--no-sparse")) { LZ4IO_setSparseFile(0); continue; }
         if (!strcmp(argument, "--verbose")) { displayLevel=4; continue; }
         if (!strcmp(argument, "--quiet")) { if (displayLevel) displayLevel--; continue; }
@@ -473,7 +473,7 @@ int main(int argc, char** argv)
     if (multiple_inputs) input_filename = inFileNames[0], output_filename = (const char*)(inFileNames[0]);
     if(!input_filename) { input_filename=stdinmark; }
 
-    /* Check if input or output are defined as console; trigger an error in this case */
+    /* Check if input is defined as console; trigger an error in this case */
     if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage();
 
     /* Check if benchmark is selected */
diff --git a/programs/lz4io.c b/programs/lz4io.c
index 209f5ed..e782664 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -278,6 +278,11 @@ static int LZ4IO_getFiles(const char* input_filename, const char* output_filenam
         DISPLAYLEVEL(4,"Using stdout for output\n");
         *pfoutput = stdout;
         SET_BINARY_MODE(stdout);
+        if (g_sparseFileSupport==1)
+        {
+            g_sparseFileSupport = 0;
+            DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n");
+        }
     }
     else
     {
@@ -301,7 +306,7 @@ static int LZ4IO_getFiles(const char* input_filename, const char* output_filenam
         *pfoutput = fopen( output_filename, "wb" );
     }
 
-    if ( *pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename);
+    if (*pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename);
 
     return 0;
 }
@@ -639,7 +644,7 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer
     if (!g_sparseFileSupport)   /* normal write */
     {
         size_t sizeCheck = fwrite(buffer, 1, bufferSize, file);
-        if (sizeCheck != bufferSize) EXM_THROW(68, "Write error : cannot write decoded block");
+        if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : cannot write decoded block");
         return 0;
     }
 
@@ -647,7 +652,7 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer
     if (storedSkips > 1 GB)
     {
         int seekResult = fseek(file, 1 GB, SEEK_CUR);
-        if (seekResult != 0) EXM_THROW(68, "1 GB skip error (sparse file support)");
+        if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)");
         storedSkips -= 1 GB;
     }
 
@@ -667,12 +672,12 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer
         {
             size_t sizeCheck;
             seekResult = fseek(file, storedSkips, SEEK_CUR);
-            if (seekResult) EXM_THROW(68, "Skip error (sparse file)");
+            if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse");
             storedSkips = 0;
             seg0SizeT -= nb0T;
             ptrT += nb0T;
             sizeCheck = fwrite(ptrT, sizeT, seg0SizeT, file);
-            if (sizeCheck != seg0SizeT) EXM_THROW(68, "Write error : cannot write decoded block");
+            if (sizeCheck != seg0SizeT) EXM_THROW(73, "Write error : cannot write decoded block");
         }
         ptrT += seg0SizeT;
     }
@@ -689,10 +694,10 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer
         {
             size_t sizeCheck;
             int seekResult = fseek(file, storedSkips, SEEK_CUR);
-            if (seekResult) EXM_THROW(68, "Skip error (end of block)");
+            if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse");
             storedSkips = 0;
             sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file);
-            if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(68, "Write error : cannot write decoded end of block");
+            if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(75, "Write error : cannot write decoded end of block");
         }
     }
 
@@ -844,7 +849,6 @@ static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE
             nextToLoad = LZ4F_decompress(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, NULL);
             if (LZ4F_isError(nextToLoad)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad));
             pos += remaining;
-            if (!nextToLoad) break;
 
             if (decodedBytes)
             {
@@ -853,6 +857,8 @@ static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE
                 DISPLAYUPDATE(2, "\rDecompressed : %u MB  ", (unsigned)(filesize>>20));
                 storedSkips = LZ4IO_fwriteSparse(dstFile, ress.dstBuffer, decodedBytes, storedSkips);
             }
+
+            if (!nextToLoad) break;
         }
     }
 
diff --git a/test/Makefile b/test/Makefile
new file mode 100644
index 0000000..d1768d4
--- /dev/null
+++ b/test/Makefile
@@ -0,0 +1,40 @@
+# ##########################################################################
+# LZ4 tests - Makefile
+# Copyright (C) Takayuki Matsuoka - Yann Collet 2011-2015
+#
+# GPL v2 License
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# You can contact the author at :
+#  - LZ4 source repository : https://github.com/Cyan4973/lz4
+#  - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
+# ##########################################################################
+# versionstest : Compatibility test between LZ4 versions stored on Github (r116+)
+# ##########################################################################
+
+PYTHON?= python3
+TESTDIR := lz4test
+
+default: all
+
+all: versionstest
+
+versionstest:
+	$(PYTHON) test-lz4-versions.py
+
+clean:
+	@rm -fR $(TESTDIR)
+	@echo Cleaning completed
diff --git a/test/test-lz4-versions.py b/test/test-lz4-versions.py
new file mode 100644
index 0000000..5531a05
--- /dev/null
+++ b/test/test-lz4-versions.py
@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+
+import glob
+import subprocess
+import filecmp
+import os
+import shutil
+import sys
+import hashlib
+
+repo_url = 'https://github.com/Cyan4973/lz4.git'
+tmp_dir_name = 'test/lz4test'
+make_cmd = 'make'
+git_cmd = 'git'
+test_dat_src = 'README.md'
+test_dat = 'test_dat'
+head = 'r999'
+
+def proc(cmd_args, pipe=True, dummy=False):
+    if dummy:
+        return
+    if pipe:
+        subproc = subprocess.Popen(cmd_args,
+                                   stdout=subprocess.PIPE, 
+                                   stderr=subprocess.PIPE)
+    else:
+        subproc = subprocess.Popen(cmd_args)
+    return subproc.communicate()
+
+def make(args, pipe=True):
+    return proc([make_cmd] + args, pipe)
+
+def git(args, pipe=True):
+    return proc([git_cmd] + args, pipe)
+
+def get_git_tags():
+    stdout, stderr = git(['tag', '-l', 'r[0-9][0-9][0-9]'])
+    tags = stdout.decode('utf-8').split()
+    return tags
+
+# http://stackoverflow.com/a/19711609/2132223
+def sha1_of_file(filepath):
+    with open(filepath, 'rb') as f:
+        return hashlib.sha1(f.read()).hexdigest()
+
+if __name__ == '__main__':
+    error_code = 0
+    base_dir = os.getcwd() + '/..'           # /path/to/lz4
+    tmp_dir = base_dir + '/' + tmp_dir_name  # /path/to/lz4/test/lz4test
+    clone_dir = tmp_dir + '/' + 'lz4'        # /path/to/lz4/test/lz4test/lz4
+    programs_dir = base_dir + '/programs'    # /path/to/lz4/programs
+    os.makedirs(tmp_dir, exist_ok=True)
+
+    # since Travis clones limited depth, we should clone full repository
+    if not os.path.isdir(clone_dir):
+        git(['clone', repo_url, clone_dir])
+
+    shutil.copy2(base_dir + '/' + test_dat_src, tmp_dir + '/' + test_dat)
+
+    # Retrieve all release tags
+    print('Retrieve all release tags :')
+    os.chdir(clone_dir)
+    tags = [head] + get_git_tags()
+    print(tags);
+
+    # Build all release lz4c and lz4c32
+    for tag in tags:
+        os.chdir(base_dir)
+        dst_lz4c   = '{}/lz4c.{}'  .format(tmp_dir, tag) # /path/to/lz4/test/lz4test/lz4c.<TAG>
+        dst_lz4c32 = '{}/lz4c32.{}'.format(tmp_dir, tag) # /path/to/lz4/test/lz4test/lz4c32.<TAG>
+        if not os.path.isfile(dst_lz4c) or not os.path.isfile(dst_lz4c32) or tag == head:
+            if tag != head:
+                r_dir = '{}/{}'.format(tmp_dir, tag)  # /path/to/lz4/test/lz4test/<TAG>
+                os.makedirs(r_dir, exist_ok=True)
+                os.chdir(clone_dir)
+                git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False)
+                os.chdir(r_dir + '/programs')  # /path/to/lz4/lz4test/<TAG>/programs
+                make(['clean', 'lz4c', 'lz4c32'], False)
+            else:
+                os.chdir(programs_dir)
+                make(['lz4c', 'lz4c32'], False)
+            shutil.copy2('lz4c',   dst_lz4c)
+            shutil.copy2('lz4c32', dst_lz4c32)
+
+    # Compress test.dat by all released lz4c and lz4c32
+    print('Compress test.dat by all released lz4c and lz4c32')
+    os.chdir(tmp_dir)
+    for lz4 in glob.glob("*.lz4"):
+        os.remove(lz4)
+    for tag in tags:
+        proc(['./lz4c.'   + tag, '-1fz', test_dat, test_dat + '_1_64_' + tag + '.lz4'])
+        proc(['./lz4c.'   + tag, '-9fz', test_dat, test_dat + '_9_64_' + tag + '.lz4'])
+        proc(['./lz4c32.' + tag, '-1fz', test_dat, test_dat + '_1_32_' + tag + '.lz4'])
+        proc(['./lz4c32.' + tag, '-9fz', test_dat, test_dat + '_9_32_' + tag + '.lz4'])
+
+    print('Full list of compressed files')
+    lz4s = sorted(glob.glob('*.lz4'))
+    for lz4 in lz4s:
+        print(lz4 + ' : ' + repr(os.path.getsize(lz4)))
+
+    # Remove duplicated .lz4 files
+    print('')
+    print('Duplicated files')
+    lz4s = sorted(glob.glob('*.lz4'))
+    for i, lz4 in enumerate(lz4s):
+        if not os.path.isfile(lz4):
+            continue
+        for j in range(i+1, len(lz4s)):
+            lz4t = lz4s[j]
+            if not os.path.isfile(lz4t):
+                continue
+            if filecmp.cmp(lz4, lz4t):
+                os.remove(lz4t)
+                print('{} == {}'.format(lz4, lz4t))
+
+    print('Enumerate only different compressed files')
+    lz4s = sorted(glob.glob('*.lz4'))
+    for lz4 in lz4s:
+        print(lz4 + ' : ' + repr(os.path.getsize(lz4)) + ', ' + sha1_of_file(lz4))
+
+    # Decompress remained .lz4 files by all released lz4c and lz4c32
+    print('Decompression tests and verifications')
+    lz4s = sorted(glob.glob('*.lz4'))
+    for dec in glob.glob("*.dec"):
+        os.remove(dec)
+    for lz4 in lz4s:
+        print(lz4, end=" ")
+        for tag in tags:
+            print(tag, end=" ")
+            proc(['./lz4c.'   + tag, '-df', lz4, lz4 + '_d64_' + tag + '.dec'])
+            proc(['./lz4c32.' + tag, '-df', lz4, lz4 + '_d32_' + tag + '.dec'])
+        print(' OK')   # well, here, decompression has worked; but file is not yet verified
+
+    # Compare all '.dec' files with test_dat
+    decs = glob.glob('*.dec')
+    for dec in decs:
+        if not filecmp.cmp(dec, test_dat):
+            print('ERR : ' + dec)
+            error_code = 1
+        else:
+            print('OK  : ' + dec)
+            os.remove(dec)
+
+    if error_code != 0:
+        print('ERROR')
+
+    sys.exit(error_code)