diff options
| -rw-r--r-- | .travis.yml | 2 | ||||
| -rw-r--r-- | Makefile | 4 | ||||
| -rw-r--r-- | NEWS | 5 | ||||
| -rw-r--r-- | README.md | 2 | ||||
| -rw-r--r-- | lib/lz4.c | 2 | ||||
| -rw-r--r-- | lib/lz4.h | 3 | ||||
| -rw-r--r-- | lz4_Block_format.md | 19 | ||||
| -rw-r--r-- | lz4_Frame_format.md | 6 | ||||
| -rw-r--r-- | programs/Makefile | 16 | ||||
| -rw-r--r-- | test/Makefile | 40 | ||||
| -rw-r--r-- | test/test-lz4-versions.py | 121 |
11 files changed, 198 insertions, 22 deletions
diff --git a/.travis.yml b/.travis.yml index 398a0f1..6c505bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ before_install: - sudo apt-get install -qq g++-multilib - sudo apt-get install -qq gcc-multilib - sudo apt-get install -qq valgrind + - sudo apt-get install -qq python3 env: - LZ4_TRAVIS_CI_ENV=travis-install @@ -18,6 +19,7 @@ env: - LZ4_TRAVIS_CI_ENV=staticAnalyze - LZ4_TRAVIS_CI_ENV=gpptest - LZ4_TRAVIS_CI_ENV=armtest + - LZ4_TRAVIS_CI_ENV=versionstest - LZ4_TRAVIS_CI_ENV=test-lz4 - LZ4_TRAVIS_CI_ENV=test-lz4c - LZ4_TRAVIS_CI_ENV=test-lz4c32 @@ -71,6 +71,7 @@ clean: @cd $(PRGDIR); $(MAKE) clean > $(VOID) @cd $(LZ4DIR); $(MAKE) clean > $(VOID) @cd examples; $(MAKE) clean > $(VOID) + @cd test; $(MAKE) clean > $(VOID) @echo Cleaning completed @@ -113,6 +114,9 @@ armtest: clean cd lib; $(MAKE) -e all CC=arm-linux-gnueabi-gcc CPPFLAGS="-Werror" cd programs; $(MAKE) -e bins CC=arm-linux-gnueabi-gcc CPPFLAGS="-Werror" +versionstest: clean + @cd test; $(MAKE) -e versionstest + streaming-examples: cd examples; $(MAKE) -e test @@ -1,5 +1,8 @@ +r130: + r129: -Added : LZ4_compress_fast() +Added : LZ4_compress_fast(), LZ4_compress_fast_continue() +Added : LZ4_compress_destSize() Changed: New lz4 and lz4hc compression API. Previous function prototypes still supported. Changed: Sparse file support enabled by default New : LZ4 CLI improved performance compressing/decompressing multiple files (#86, kind contribution from Kyle J. Harper & Takayuki Matsuoka) @@ -77,4 +77,4 @@ A list of these sources is maintained on the [LZ4 Homepage]. [Silesia Corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia [lz4_Block_format]: lz4_Block_format.md [lz4_Frame_format]: lz4_Frame_format.md -[LZ4 Homepage]: http://www.lz4.info +[LZ4 Homepage]: http://www.lz4.org @@ -45,7 +45,7 @@ /* * ACCELERATION_DEFAULT : - * Select the value of "acceleration" for LZ4_compress_fast() when parameter == 0 + * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0 */ #define ACCELERATION_DEFAULT 1 @@ -122,8 +122,8 @@ LZ4_compress_fast() : Same as LZ4_compress_default(), but allows to select an "acceleration" factor. The larger the acceleration value, the faster the algorithm, but also the lesser the compression. It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. - An acceleration value of "0" means "use Default value" (see lz4.c) An acceleration value of "1" is the same as regular LZ4_compress_default() + Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1. */ int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration); @@ -145,7 +145,6 @@ LZ4_compress_destSize() : into already allocated buffer 'dest' of size 'targetDestSize'. This function either compresses the entire 'source' content into 'dest' if it's large enough, or fill 'dest' buffer completely with as much data as possible from 'source'. - Original idea by WiredTiger team. *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'. New value is necessarily <= old value. return : Nb bytes written into 'dest' (necessarily <= targetDestSize) diff --git a/lz4_Block_format.md b/lz4_Block_format.md index a120a0b..ea568d8 100644 --- a/lz4_Block_format.md +++ b/lz4_Block_format.md @@ -1,25 +1,26 @@ LZ4 Block Format Description ============================ -Last revised: 2015-03-26. +Last revised: 2015-05-07. Author : Yann Collet -This small specification intents to provide enough information -to anyone willing to produce LZ4-compatible compressed data blocks +This specification is intended for developers +willing to produce LZ4-compatible compressed data blocks using any programming language. LZ4 is an LZ77-type compressor with a fixed, byte-oriented encoding. -The most important design principle behind LZ4 is simplicity. -It helps to create an easy to read and maintain source code. -It also helps later on for optimizations, compactness, and speed. There is no entropy encoder back-end nor framing layer. -The latter is assumed to be handled by other parts of the system. +The latter is assumed to be handled by other parts of the system (see [LZ4 Frame format]). +This design is assumed to favor simplicity and speed. +It helps later on for optimizations, compactness, and features. -This document only describes the block format, -not how the LZ4 compressor nor decompressor actually work. +This document describes only the block format, +not how the compressor nor decompressor actually work. The correctness of the decompressor should not depend on implementation details of the compressor, and vice versa. +[LZ4 Frame format]: LZ4_Frame_format.md + Compressed block format diff --git a/lz4_Frame_format.md b/lz4_Frame_format.md index cb41d47..73d3cba 100644 --- a/lz4_Frame_format.md +++ b/lz4_Frame_format.md @@ -205,13 +205,13 @@ This is the original (uncompressed) size. This information is optional, and only present if the associated flag is set. Content size is provided using unsigned 8 Bytes, for a maximum of 16 HexaBytes. Format is Little endian. -This value is informational. -It can be safely skipped by a conformant decoder. +This value is informational, typically for display or memory allocation. +It can be skipped by a decoder, or used to validate content correctness. __Header Checksum__ One-byte checksum of combined descriptor fields, including optional ones. -The value is the second byte of xxh32() : ` (xxh32()>>8) & 0xFF } ` +The value is the second byte of xxh32() : ` (xxh32()>>8) & 0xFF ` using zero as a seed, and the full Frame Descriptor as an input (including optional fields when they are present). diff --git a/programs/Makefile b/programs/Makefile index 39335db..8ede048 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -224,16 +224,22 @@ test-lz4: lz4 datagen test-lz4-multiple test-lz4-sparse test-lz4-contentSize tes test-lz4c: lz4c datagen ./datagen -g256MB | ./lz4c -l -v | ./lz4c -t -test-lz4c32: lz4 lz4c32 datagen - ./datagen -g16KB | ./lz4c32 -9 | ./lz4c32 -t +test-interop-32-64: lz4 lz4c32 datagen + @echo ---- test interoperability 32-bits -vs- 64 bits ---- ./datagen -g16KB | ./lz4c32 -9 | ./lz4 -t - ./datagen | ./lz4c32 | ./lz4c32 -t + ./datagen -P10 | ./lz4 -9B4 | ./lz4c32 -t ./datagen | ./lz4c32 | ./lz4 -t - ./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4c32 -qt + ./datagen -g1M | ./lz4 -3B5 | ./lz4c32 -t ./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4 -qt - ./datagen -g6GB | ./lz4c32 -vqB5D | ./lz4c32 -qt + ./datagen -g1G -P90 | ./lz4 | ./lz4c32 -t ./datagen -g6GB | ./lz4c32 -vq9BD | ./lz4 -qt +test-lz4c32: lz4c32 datagen test-interop-32-64 + ./datagen -g16KB | ./lz4c32 -9 | ./lz4c32 -t + ./datagen | ./lz4c32 | ./lz4c32 -t + ./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4c32 -qt + ./datagen -g6GB | ./lz4c32 -vqB5D | ./lz4c32 -qt + test-fullbench: fullbench ./fullbench --no-prompt $(NB_LOOPS) $(TEST_FILES) diff --git a/test/Makefile b/test/Makefile new file mode 100644 index 0000000..af377c4 --- /dev/null +++ b/test/Makefile @@ -0,0 +1,40 @@ +# ########################################################################## +# LZ4 tests - Makefile +# Copyright (C) Yann Collet 2011-2015 +# +# GPL v2 License +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# You can contact the author at : +# - LZ4 source repository : https://github.com/Cyan4973/lz4 +# - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +# ########################################################################## +# versionstest : Compatibility test between all LZ4 versions +# ########################################################################## + +PYTHON?= python3 +TESTDIR := lz4test + +default: all + +all: versionstest + +versionstest: + $(PYTHON) test-lz4-versions.py + +clean: + @rm -fR $(TESTDIR) + @echo Cleaning completed diff --git a/test/test-lz4-versions.py b/test/test-lz4-versions.py new file mode 100644 index 0000000..6605051 --- /dev/null +++ b/test/test-lz4-versions.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 + +import glob +import subprocess +import filecmp +import os +import shutil +import sys + +repo_url = 'https://github.com/Cyan4973/lz4.git' +tmp_dir_name = 'test/lz4test' +make_cmd = 'make' +git_cmd = 'git' +test_dat_src = 'README.md' +test_dat = 'test_dat' +head = 'r999' + +def proc(cmd_args, pipe=True, dummy=False): + if dummy: + return + if pipe: + subproc = subprocess.Popen(cmd_args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + else: + subproc = subprocess.Popen(cmd_args) + return subproc.communicate() + +def make(args, pipe=True): + return proc([make_cmd] + args, pipe) + +def git(args, pipe=True): + return proc([git_cmd] + args, pipe) + +def get_git_tags(): + stdout, stderr = git(['tag', '-l', 'r[0-9][0-9][0-9]']) + tags = stdout.decode('utf-8').split() + return tags + +if __name__ == '__main__': + error_code = 0 + base_dir = os.getcwd() + '/..' # /path/to/lz4 + tmp_dir = base_dir + '/' + tmp_dir_name # /path/to/lz4/test/lz4test + clone_dir = tmp_dir + '/' + 'lz4' # /path/to/lz4/test/lz4test/lz4 + programs_dir = base_dir + '/programs' # /path/to/lz4/programs + os.makedirs(tmp_dir, exist_ok=True) + + # since Travis clones limited depth, we should clone full repository + if not os.path.isdir(clone_dir): + git(['clone', repo_url, clone_dir]) + + shutil.copy2(base_dir + '/' + test_dat_src, tmp_dir + '/' + test_dat) + + # Retrieve all release tags + os.chdir(clone_dir) + tags = [head] + get_git_tags() + + # Build all release lz4c and lz4c32 + for tag in tags: + os.chdir(base_dir) + dst_lz4c = '{}/lz4c.{}' .format(tmp_dir, tag) # /path/to/lz4/test/lz4test/lz4c.<TAG> + dst_lz4c32 = '{}/lz4c32.{}'.format(tmp_dir, tag) # /path/to/lz4/test/lz4test/lz4c32.<TAG> + if not os.path.isfile(dst_lz4c) or not os.path.isfile(dst_lz4c32): + if tag != head: + r_dir = '{}/{}'.format(tmp_dir, tag) # /path/to/lz4/test/lz4test/<TAG> + os.makedirs(r_dir, exist_ok=True) + os.chdir(clone_dir) + git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False) + os.chdir(r_dir + '/programs') # /path/to/lz4/lz4test/<TAG>/programs + else: + os.chdir(programs_dir) + make(['clean', 'lz4c', 'lz4c32'], False) + shutil.copy2('lz4c', dst_lz4c) + shutil.copy2('lz4c32', dst_lz4c32) + + # Compress test.dat by all released lz4c and lz4c32 + os.chdir(tmp_dir) + for lz4 in glob.glob("*.lz4"): + os.remove(lz4) + for tag in tags: + proc(['./lz4c.' + tag, '-1fz', test_dat, test_dat + '_1_64_' + tag + '.lz4']) + proc(['./lz4c.' + tag, '-9fz', test_dat, test_dat + '_9_64_' + tag + '.lz4']) + proc(['./lz4c32.' + tag, '-1fz', test_dat, test_dat + '_1_32_' + tag + '.lz4']) + proc(['./lz4c32.' + tag, '-9fz', test_dat, test_dat + '_9_32_' + tag + '.lz4']) + + # Remove duplicated .lz4 files + lz4s = sorted(glob.glob('*.lz4')) + for i, lz4 in enumerate(lz4s): + if not os.path.isfile(lz4): + continue + for j in range(i+1, len(lz4s)): + lz4t = lz4s[j] + if not os.path.isfile(lz4t): + continue + if filecmp.cmp(lz4, lz4t): + os.remove(lz4t) + + # Decompress remained .lz4 files by all released lz4c and lz4c32 + lz4s = sorted(glob.glob('*.lz4')) + for lz4 in lz4s: + for tag in tags: + proc(['./lz4c.' + tag, '-df', lz4, lz4 + '_d64_' + tag + '.dec']) + proc(['./lz4c32.' + tag, '-df', lz4, lz4 + '_d32_' + tag + '.dec']) + + # Compare all '.dec' files with test_dat + decs = glob.glob('*.dec') + for dec in decs: + if not filecmp.cmp(dec, test_dat): + print('ERR : ' + dec) + error_code = 1 + else: + os.remove(dec) + + lz4s = sorted(glob.glob('*.lz4')) + for lz4 in lz4s: + print(lz4) + + if error_code != 0: + print('ERROR') + + sys.exit(error_code) |