Issue #103 : Add compatibility test between releases

Add forward/backward compatibility test between all versions

 - Add new Makefile target "versionstest".
 - Standard test procedure like "make versionstest && make clean"
   works properly.

test/test-lz4-versions.py

 - This script creates test/lz4test and checkout all
   Release tagged versions.  After that, compile all versions of
   lz4c and lz4c32 and compress/decompress between all versions.

 - This test clones entire git repository to checkout all tags.
    - Since Travis's default command does not clone entire repository,
      we could not checkout all tags from Travis' default clone.

 - After that, test script shows all unique .lz4 files.  In this list,
   'r999' means 'head' of current working copy.

4 files changed, 167 insertions, 0 deletions

diff --git a/.travis.yml b/.travis.yml
index 398a0f1..6c505bc 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,6 +8,7 @@ before_install:
   - sudo apt-get install -qq g++-multilib
   - sudo apt-get install -qq gcc-multilib
   - sudo apt-get install -qq valgrind
+  - sudo apt-get install -qq python3
 
 env:
   - LZ4_TRAVIS_CI_ENV=travis-install
@@ -18,6 +19,7 @@ env:
   - LZ4_TRAVIS_CI_ENV=staticAnalyze
   - LZ4_TRAVIS_CI_ENV=gpptest
   - LZ4_TRAVIS_CI_ENV=armtest
+  - LZ4_TRAVIS_CI_ENV=versionstest
   - LZ4_TRAVIS_CI_ENV=test-lz4
   - LZ4_TRAVIS_CI_ENV=test-lz4c
   - LZ4_TRAVIS_CI_ENV=test-lz4c32
diff --git a/Makefile b/Makefile
index 88c4016..93cf846 100644
--- a/Makefile
+++ b/Makefile
@@ -71,6 +71,7 @@ clean:
 	@cd $(PRGDIR); $(MAKE) clean > $(VOID)
 	@cd $(LZ4DIR); $(MAKE) clean > $(VOID)
 	@cd examples;  $(MAKE) clean > $(VOID)
+	@cd test;      $(MAKE) clean > $(VOID)
 	@echo Cleaning completed
 
 
@@ -113,6 +114,9 @@ armtest: clean
 	cd lib; $(MAKE) -e all CC=arm-linux-gnueabi-gcc CPPFLAGS="-Werror"
 	cd programs; $(MAKE) -e bins CC=arm-linux-gnueabi-gcc CPPFLAGS="-Werror"
 
+versionstest: clean
+	@cd test; $(MAKE) -e versionstest
+
 streaming-examples:
 	cd examples; $(MAKE) -e test
 
diff --git a/test/Makefile b/test/Makefile
new file mode 100644
index 0000000..af377c4
--- /dev/null
+++ b/test/Makefile
@@ -0,0 +1,40 @@
+# ##########################################################################
+# LZ4 tests - Makefile
+# Copyright (C) Yann Collet 2011-2015
+#
+# GPL v2 License
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# You can contact the author at :
+#  - LZ4 source repository : https://github.com/Cyan4973/lz4
+#  - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
+# ##########################################################################
+# versionstest : Compatibility test between all LZ4 versions
+# ##########################################################################
+
+PYTHON?= python3
+TESTDIR := lz4test
+
+default: all
+
+all: versionstest
+
+versionstest:
+	$(PYTHON) test-lz4-versions.py
+
+clean:
+	@rm -fR $(TESTDIR)
+	@echo Cleaning completed
diff --git a/test/test-lz4-versions.py b/test/test-lz4-versions.py
new file mode 100644
index 0000000..6605051
--- /dev/null
+++ b/test/test-lz4-versions.py
@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+
+import glob
+import subprocess
+import filecmp
+import os
+import shutil
+import sys
+
+repo_url = 'https://github.com/Cyan4973/lz4.git'
+tmp_dir_name = 'test/lz4test'
+make_cmd = 'make'
+git_cmd = 'git'
+test_dat_src = 'README.md'
+test_dat = 'test_dat'
+head = 'r999'
+
+def proc(cmd_args, pipe=True, dummy=False):
+    if dummy:
+        return
+    if pipe:
+        subproc = subprocess.Popen(cmd_args,
+                                   stdout=subprocess.PIPE, 
+                                   stderr=subprocess.PIPE)
+    else:
+        subproc = subprocess.Popen(cmd_args)
+    return subproc.communicate()
+
+def make(args, pipe=True):
+    return proc([make_cmd] + args, pipe)
+
+def git(args, pipe=True):
+    return proc([git_cmd] + args, pipe)
+
+def get_git_tags():
+    stdout, stderr = git(['tag', '-l', 'r[0-9][0-9][0-9]'])
+    tags = stdout.decode('utf-8').split()
+    return tags
+
+if __name__ == '__main__':
+    error_code = 0
+    base_dir = os.getcwd() + '/..'           # /path/to/lz4
+    tmp_dir = base_dir + '/' + tmp_dir_name  # /path/to/lz4/test/lz4test
+    clone_dir = tmp_dir + '/' + 'lz4'        # /path/to/lz4/test/lz4test/lz4
+    programs_dir = base_dir + '/programs'    # /path/to/lz4/programs
+    os.makedirs(tmp_dir, exist_ok=True)
+
+    # since Travis clones limited depth, we should clone full repository
+    if not os.path.isdir(clone_dir):
+        git(['clone', repo_url, clone_dir])
+
+    shutil.copy2(base_dir + '/' + test_dat_src, tmp_dir + '/' + test_dat)
+
+    # Retrieve all release tags
+    os.chdir(clone_dir)
+    tags = [head] + get_git_tags()
+
+    # Build all release lz4c and lz4c32
+    for tag in tags:
+        os.chdir(base_dir)
+        dst_lz4c   = '{}/lz4c.{}'  .format(tmp_dir, tag) # /path/to/lz4/test/lz4test/lz4c.<TAG>
+        dst_lz4c32 = '{}/lz4c32.{}'.format(tmp_dir, tag) # /path/to/lz4/test/lz4test/lz4c32.<TAG>
+        if not os.path.isfile(dst_lz4c) or not os.path.isfile(dst_lz4c32):
+            if tag != head:
+                r_dir = '{}/{}'.format(tmp_dir, tag)  # /path/to/lz4/test/lz4test/<TAG>
+                os.makedirs(r_dir, exist_ok=True)
+                os.chdir(clone_dir)
+                git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False)
+                os.chdir(r_dir + '/programs')  # /path/to/lz4/lz4test/<TAG>/programs
+            else:
+                os.chdir(programs_dir)
+            make(['clean', 'lz4c', 'lz4c32'], False)
+            shutil.copy2('lz4c',   dst_lz4c)
+            shutil.copy2('lz4c32', dst_lz4c32)
+
+    # Compress test.dat by all released lz4c and lz4c32
+    os.chdir(tmp_dir)
+    for lz4 in glob.glob("*.lz4"):
+        os.remove(lz4)
+    for tag in tags:
+        proc(['./lz4c.'   + tag, '-1fz', test_dat, test_dat + '_1_64_' + tag + '.lz4'])
+        proc(['./lz4c.'   + tag, '-9fz', test_dat, test_dat + '_9_64_' + tag + '.lz4'])
+        proc(['./lz4c32.' + tag, '-1fz', test_dat, test_dat + '_1_32_' + tag + '.lz4'])
+        proc(['./lz4c32.' + tag, '-9fz', test_dat, test_dat + '_9_32_' + tag + '.lz4'])
+
+    # Remove duplicated .lz4 files
+    lz4s = sorted(glob.glob('*.lz4'))
+    for i, lz4 in enumerate(lz4s):
+        if not os.path.isfile(lz4):
+            continue
+        for j in range(i+1, len(lz4s)):
+            lz4t = lz4s[j]
+            if not os.path.isfile(lz4t):
+                continue
+            if filecmp.cmp(lz4, lz4t):
+                os.remove(lz4t)
+
+    # Decompress remained .lz4 files by all released lz4c and lz4c32
+    lz4s = sorted(glob.glob('*.lz4'))
+    for lz4 in lz4s:
+        for tag in tags:
+            proc(['./lz4c.'   + tag, '-df', lz4, lz4 + '_d64_' + tag + '.dec'])
+            proc(['./lz4c32.' + tag, '-df', lz4, lz4 + '_d32_' + tag + '.dec'])
+
+    # Compare all '.dec' files with test_dat
+    decs = glob.glob('*.dec')
+    for dec in decs:
+        if not filecmp.cmp(dec, test_dat):
+            print('ERR : ' + dec)
+            error_code = 1
+        else:
+            os.remove(dec)
+
+    lz4s = sorted(glob.glob('*.lz4'))
+    for lz4 in lz4s:
+        print(lz4)
+
+    if error_code != 0:
+        print('ERROR')
+
+    sys.exit(error_code)