From 207f478babe676fc000e6e655c150a03a79570b0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 4 Nov 2016 19:04:32 -0700 Subject: minor refactor --- NEWS | 1 + lib/lz4.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 7bfa3c5..9cd080e 100644 --- a/NEWS +++ b/NEWS @@ -5,6 +5,7 @@ Improved: Small compression speed improvement on 64-bits systems Improved: Performance on ARMv6 and ARMv7 Added : Debianization, by Evgeniy Polyakov Makefile: Generates object files (*.o) for faster (re)compilation on low power systems +New : recursive mode in benchmark, by Przemyslaw Skibinski Fix : cli : crash on some invalid inputs Fix : cli : -t correctly validates lz4-compressed files, by Nick Terrell Fix : better ratio on 64-bits big-endian targets diff --git a/lib/lz4.c b/lib/lz4.c index 6db4c0b..39a7de9 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -701,7 +701,7 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ #else LZ4_stream_t ctx; - void* ctxPtr = &ctx; + void* const ctxPtr = &ctx; #endif int result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); -- cgit v0.12 From 2fbd7a5120da09dd520034b98830b7230c3bf6e3 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Mon, 7 Nov 2016 16:15:22 +0100 Subject: updated travis.yml --- .travis.yml | 56 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.travis.yml b/.travis.yml index ef8732c..19b24eb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,57 +4,57 @@ matrix: include: # OS X Mavericks - os: osx - env: Ubu="OS X Mavericks" PARAMS='-C tests test-lz4 CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"' COMPILER=clang + env: Ubu="OS X Mavericks" CMD='-C tests test-lz4 CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"' COMPILER=clang # Container-based 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - os: linux sudo: false - env: Ubu="12.04cont" PARAMS="install PREFIX=~/install_test_dir" COMPILER=cc + env: Ubu="12.04cont" CMD="make install PREFIX=~/install_test_dir" COMPILER=cc - os: linux sudo: false - env: Ubu="12.04cont" PARAMS="examples" COMPILER=cc + env: Ubu="12.04cont" CMD="make examples" COMPILER=cc - os: linux sudo: false - env: Ubu="12.04cont" PARAMS="cmake" COMPILER=cc + env: Ubu="12.04cont" CMD="make cmake" COMPILER=cc - os: linux sudo: false - env: Ubu="12.04cont" PARAMS=clangtest COMPILER=clang + env: Ubu="12.04cont" CMD='make clangtest' COMPILER=clang - os: linux sudo: false - env: Ubu="12.04cont" PARAMS=gpptest COMPILER=cc + env: Ubu="12.04cont" CMD='make gpptest' COMPILER=cc - os: linux sudo: false - env: Ubu="12.04cont" PARAMS='-C tests test-lz4' COMPILER=cc + env: Ubu="12.04cont" CMD='make -C tests test-lz4' COMPILER=cc - os: linux sudo: false - env: Ubu="12.04cont" PARAMS='-C tests test-lz4 MOREFLAGS="-std=c99"' COMPILER=cc + env: Ubu="12.04cont" CMD='make -C tests test-lz4 MOREFLAGS="-std=c99"' COMPILER=cc - os: linux sudo: false - env: Ubu="12.04cont" PARAMS='-C tests test-lz4c' COMPILER=cc + env: Ubu="12.04cont" CMD='make -C tests test-lz4c' COMPILER=cc - os: linux sudo: false - env: Ubu="12.04cont" PARAMS='-C tests test-fullbench' COMPILER=cc + env: Ubu="12.04cont" CMD='make -C tests test-fullbench' COMPILER=cc - os: linux sudo: false - env: Ubu="12.04cont" PARAMS='-C tests test-fuzzer' COMPILER=cc + env: Ubu="12.04cont" CMD='make -C tests test-fuzzer' COMPILER=cc - os: linux sudo: false - env: Ubu="12.04cont" PARAMS='-C tests test-frametest' COMPILER=cc + env: Ubu="12.04cont" CMD='make -C tests test-frametest' COMPILER=cc # 14.04 LTS Server Edition 64 bit - - env: Ubu="14.04" PARAMS=sanitize COMPILER=clang + - env: Ubu="14.04" CMD='make sanitize' COMPILER=clang dist: trusty sudo: required addons: apt: packages: - valgrind - - env: Ubu="14.04" PARAMS=staticAnalyze COMPILER=clang + - env: Ubu="14.04" CMD='make staticAnalyze' COMPILER=clang dist: trusty sudo: required addons: apt: packages: - clang - - env: Ubu="14.04" PARAMS=armtest COMPILER=arm-linux-gnueabi-gcc + - env: Ubu="14.04" CMD='make armtest' COMPILER=arm-linux-gnueabi-gcc dist: trusty sudo: required addons: @@ -63,7 +63,7 @@ matrix: - gcc-arm-linux-gnueabi - libc6-dev-armel-cross - linux-libc-dev-armel-cross - - env: Ubu="14.04" PARAMS=versionsTest COMPILER=cc + - env: Ubu="14.04" CMD='make versionsTest' COMPILER=cc dist: trusty sudo: required addons: @@ -72,10 +72,10 @@ matrix: - python3 - libc6-dev-i386 - gcc-multilib - - env: Ubu="14.04" PARAMS='-C tests test-lz4' COMPILER=cc + - env: Ubu="14.04" CMD='make -C tests test-lz4' COMPILER=cc dist: trusty sudo: required - - env: Ubu="14.04" PARAMS='-C tests test-lz4c32' COMPILER=cc + - env: Ubu="14.04" CMD='make -C tests test-lz4c32' COMPILER=cc dist: trusty sudo: required addons: @@ -83,7 +83,7 @@ matrix: packages: - libc6-dev-i386 - gcc-multilib - - env: Ubu="14.04" PARAMS='-C tests test-lz4 CC=gcc-5' COMPILER=gcc-5 + - env: Ubu="14.04" CMD='make -C tests test-lz4 CC=gcc-5' COMPILER=gcc-5 dist: trusty sudo: required addons: @@ -92,7 +92,7 @@ matrix: - ubuntu-toolchain-r-test packages: - gcc-5 - - env: Ubu="14.04" PARAMS='-C tests test-lz4c32 CC=gcc-5' COMPILER=gcc-5 + - env: Ubu="14.04" CMD='make -C tests test-lz4c32 CC=gcc-5' COMPILER=gcc-5 dist: trusty sudo: required addons: @@ -104,7 +104,7 @@ matrix: - gcc-multilib - gcc-5 - gcc-5-multilib - - env: Ubu="14.04" PARAMS='-C tests test-lz4 CC=gcc-6' COMPILER=gcc-6 + - env: Ubu="14.04" CMD='make -C tests test-lz4 CC=gcc-6' COMPILER=gcc-6 dist: trusty sudo: required addons: @@ -113,7 +113,7 @@ matrix: - ubuntu-toolchain-r-test packages: - gcc-6 - - env: Ubu="14.04" PARAMS='-C tests test-lz4 CC=clang-3.8' COMPILER=clang-3.8 + - env: Ubu="14.04" CMD='make -C tests test-lz4 CC=clang-3.8' COMPILER=clang-3.8 dist: trusty sudo: required addons: @@ -123,7 +123,7 @@ matrix: - llvm-toolchain-precise-3.8 packages: - clang-3.8 - - env: Ubu="14.04" PARAMS='-C tests test-fullbench32' COMPILER=cc + - env: Ubu="14.04" CMD='make -C tests test-fullbench32' COMPILER=cc dist: trusty sudo: required addons: @@ -131,7 +131,7 @@ matrix: packages: - libc6-dev-i386 - gcc-multilib - - env: Ubu="14.04" PARAMS='-C tests test-fuzzer32' COMPILER=cc + - env: Ubu="14.04" CMD='make -C tests test-fuzzer32' COMPILER=cc dist: trusty sudo: required addons: @@ -139,7 +139,7 @@ matrix: packages: - libc6-dev-i386 - gcc-multilib - - env: Ubu="14.04" PARAMS='-C tests test-frametest32' COMPILER=cc + - env: Ubu="14.04" CMD='make -C tests test-frametest32' COMPILER=cc dist: trusty sudo: required addons: @@ -147,7 +147,7 @@ matrix: packages: - libc6-dev-i386 - gcc-multilib - - env: Ubu="14.04" PARAMS='-C tests test-mem' COMPILER=cc + - env: Ubu="14.04" CMD='make -C tests test-mem' COMPILER=cc dist: trusty sudo: required addons: @@ -156,6 +156,6 @@ matrix: - valgrind script: - - echo PARAMS=$PARAMS + - echo CMD=$CMD - $COMPILER -v - - sh -c "make $PARAMS" + - sh -c "$CMD" -- cgit v0.12 From b6c407886f6c09125478c5b8b633a26e6e3dd9b8 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Mon, 7 Nov 2016 16:46:39 +0100 Subject: condensed travis.yml --- .travis.yml | 86 +++++-------------------------------------------------------- 1 file changed, 6 insertions(+), 80 deletions(-) diff --git a/.travis.yml b/.travis.yml index 19b24eb..27b4a1f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,55 +4,25 @@ matrix: include: # OS X Mavericks - os: osx - env: Ubu="OS X Mavericks" CMD='-C tests test-lz4 CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"' COMPILER=clang + env: Ubu="OS X Mavericks" CMD='make -C tests test-lz4 CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"' COMPILER=clang # Container-based 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - os: linux sudo: false - env: Ubu="12.04cont" CMD="make install PREFIX=~/install_test_dir" COMPILER=cc - - os: linux - sudo: false - env: Ubu="12.04cont" CMD="make examples" COMPILER=cc - - os: linux - sudo: false - env: Ubu="12.04cont" CMD="make cmake" COMPILER=cc + env: Ubu="12.04cont" CMD="make gpptest && make clean && make examples && make clean && make cmake && make clean && make travis-install" COMPILER=cc - os: linux sudo: false env: Ubu="12.04cont" CMD='make clangtest' COMPILER=clang - os: linux sudo: false - env: Ubu="12.04cont" CMD='make gpptest' COMPILER=cc - - os: linux - sudo: false - env: Ubu="12.04cont" CMD='make -C tests test-lz4' COMPILER=cc - - os: linux - sudo: false - env: Ubu="12.04cont" CMD='make -C tests test-lz4 MOREFLAGS="-std=c99"' COMPILER=cc - - os: linux - sudo: false - env: Ubu="12.04cont" CMD='make -C tests test-lz4c' COMPILER=cc - - os: linux - sudo: false - env: Ubu="12.04cont" CMD='make -C tests test-fullbench' COMPILER=cc - - os: linux - sudo: false - env: Ubu="12.04cont" CMD='make -C tests test-fuzzer' COMPILER=cc - - os: linux - sudo: false - env: Ubu="12.04cont" CMD='make -C tests test-frametest' COMPILER=cc + env: Ubu="12.04cont" CMD='make -C tests test-lz4 && make clean && make -C tests test-lz4 MOREFLAGS="-std=c99" && make clean && make -C tests test-lz4c test-fullbench test-fuzzer test-frametest' COMPILER=cc # 14.04 LTS Server Edition 64 bit - - env: Ubu="14.04" CMD='make sanitize' COMPILER=clang + - env: Ubu="14.04" CMD='make staticAnalyze sanitize' COMPILER=clang dist: trusty sudo: required addons: apt: packages: - valgrind - - env: Ubu="14.04" CMD='make staticAnalyze' COMPILER=clang - dist: trusty - sudo: required - addons: - apt: - packages: - clang - env: Ubu="14.04" CMD='make armtest' COMPILER=arm-linux-gnueabi-gcc dist: trusty @@ -63,7 +33,7 @@ matrix: - gcc-arm-linux-gnueabi - libc6-dev-armel-cross - linux-libc-dev-armel-cross - - env: Ubu="14.04" CMD='make versionsTest' COMPILER=cc + - env: Ubu="14.04" CMD='make -C tests test-lz4 && make clean && make -C tests test-lz4c32 test-fullbench32 test-fuzzer32 test-frametest32 versionsTest' COMPILER=cc dist: trusty sudo: required addons: @@ -72,27 +42,7 @@ matrix: - python3 - libc6-dev-i386 - gcc-multilib - - env: Ubu="14.04" CMD='make -C tests test-lz4' COMPILER=cc - dist: trusty - sudo: required - - env: Ubu="14.04" CMD='make -C tests test-lz4c32' COMPILER=cc - dist: trusty - sudo: required - addons: - apt: - packages: - - libc6-dev-i386 - - gcc-multilib - - env: Ubu="14.04" CMD='make -C tests test-lz4 CC=gcc-5' COMPILER=gcc-5 - dist: trusty - sudo: required - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-5 - - env: Ubu="14.04" CMD='make -C tests test-lz4c32 CC=gcc-5' COMPILER=gcc-5 + - env: Ubu="14.04" CMD='make -C tests test-lz4 test-lz4c32 CC=gcc-5' COMPILER=gcc-5 dist: trusty sudo: required addons: @@ -123,30 +73,6 @@ matrix: - llvm-toolchain-precise-3.8 packages: - clang-3.8 - - env: Ubu="14.04" CMD='make -C tests test-fullbench32' COMPILER=cc - dist: trusty - sudo: required - addons: - apt: - packages: - - libc6-dev-i386 - - gcc-multilib - - env: Ubu="14.04" CMD='make -C tests test-fuzzer32' COMPILER=cc - dist: trusty - sudo: required - addons: - apt: - packages: - - libc6-dev-i386 - - gcc-multilib - - env: Ubu="14.04" CMD='make -C tests test-frametest32' COMPILER=cc - dist: trusty - sudo: required - addons: - apt: - packages: - - libc6-dev-i386 - - gcc-multilib - env: Ubu="14.04" CMD='make -C tests test-mem' COMPILER=cc dist: trusty sudo: required -- cgit v0.12 From 7fbb163f3fe39920f3c0ba96a74e9af26727f48c Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Mon, 7 Nov 2016 17:43:37 +0100 Subject: added "make c_standards" --- .travis.yml | 40 +++++++++++++++++++++------------------- Makefile | 24 +++++++++++++++++------- tests/Makefile | 4 ++-- 3 files changed, 40 insertions(+), 28 deletions(-) diff --git a/.travis.yml b/.travis.yml index 27b4a1f..4a18b17 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,19 +4,21 @@ matrix: include: # OS X Mavericks - os: osx - env: Ubu="OS X Mavericks" CMD='make -C tests test-lz4 CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"' COMPILER=clang + env: Ubu=OS_X_Mavericks Cmd='make -C tests test-lz4 CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"' COMPILER=clang + # Container-based 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - os: linux sudo: false - env: Ubu="12.04cont" CMD="make gpptest && make clean && make examples && make clean && make cmake && make clean && make travis-install" COMPILER=cc + env: Ubu=12.04cont Cmd="make gpptest && make clean && make examples && make clean && make cmake && make clean && make travis-install" COMPILER=cc - os: linux sudo: false - env: Ubu="12.04cont" CMD='make clangtest' COMPILER=clang + env: Ubu=12.04cont Cmd='make test && make clean' COMPILER=cc - os: linux sudo: false - env: Ubu="12.04cont" CMD='make -C tests test-lz4 && make clean && make -C tests test-lz4 MOREFLAGS="-std=c99" && make clean && make -C tests test-lz4c test-fullbench test-fuzzer test-frametest' COMPILER=cc + env: Ubu=12.04cont Cmd='make clangtest' COMPILER=clang + # 14.04 LTS Server Edition 64 bit - - env: Ubu="14.04" CMD='make staticAnalyze sanitize' COMPILER=clang + - env: Ubu=14.04 Cmd='make staticAnalyze sanitize' COMPILER=clang dist: trusty sudo: required addons: @@ -24,7 +26,7 @@ matrix: packages: - valgrind - clang - - env: Ubu="14.04" CMD='make armtest' COMPILER=arm-linux-gnueabi-gcc + - env: Ubu=14.04 Cmd='make armtest' COMPILER=arm-linux-gnueabi-gcc dist: trusty sudo: required addons: @@ -33,7 +35,14 @@ matrix: - gcc-arm-linux-gnueabi - libc6-dev-armel-cross - linux-libc-dev-armel-cross - - env: Ubu="14.04" CMD='make -C tests test-lz4 && make clean && make -C tests test-lz4c32 test-fullbench32 test-fuzzer32 test-frametest32 versionsTest' COMPILER=cc + - env: Ubu=14.04 Cmd='make c_standards && make -C tests test-lz4 test-mem' COMPILER=cc + dist: trusty + sudo: required + addons: + apt: + packages: + - valgrind + - env: Ubu=14.04 Cmd='make -C tests test32 versionsTest' COMPILER=cc dist: trusty sudo: required addons: @@ -42,7 +51,7 @@ matrix: - python3 - libc6-dev-i386 - gcc-multilib - - env: Ubu="14.04" CMD='make -C tests test-lz4 test-lz4c32 CC=gcc-5' COMPILER=gcc-5 + - env: Ubu=14.04 Cmd='make -C tests test-lz4 test-lz4c32 CC=gcc-5' COMPILER=gcc-5 dist: trusty sudo: required addons: @@ -54,7 +63,7 @@ matrix: - gcc-multilib - gcc-5 - gcc-5-multilib - - env: Ubu="14.04" CMD='make -C tests test-lz4 CC=gcc-6' COMPILER=gcc-6 + - env: Ubu=14.04 Cmd='make c_standards && make -C tests test-lz4 CC=gcc-6' COMPILER=gcc-6 dist: trusty sudo: required addons: @@ -63,7 +72,7 @@ matrix: - ubuntu-toolchain-r-test packages: - gcc-6 - - env: Ubu="14.04" CMD='make -C tests test-lz4 CC=clang-3.8' COMPILER=clang-3.8 + - env: Ubu=14.04 Cmd='make -C tests test-lz4 CC=clang-3.8' COMPILER=clang-3.8 dist: trusty sudo: required addons: @@ -73,15 +82,8 @@ matrix: - llvm-toolchain-precise-3.8 packages: - clang-3.8 - - env: Ubu="14.04" CMD='make -C tests test-mem' COMPILER=cc - dist: trusty - sudo: required - addons: - apt: - packages: - - valgrind script: - - echo CMD=$CMD + - echo Cmd=$Cmd - $COMPILER -v - - sh -c "$CMD" + - sh -c "$Cmd" diff --git a/Makefile b/Makefile index e98aafd..aeb2f4b 100644 --- a/Makefile +++ b/Makefile @@ -85,7 +85,7 @@ uninstall: @$(MAKE) -C $(PRGDIR) $@ travis-install: - sudo $(MAKE) install + $(MAKE) install PREFIX=~/install_test_dir test: $(MAKE) -C $(TESTDIR) test @@ -96,19 +96,29 @@ cmake: gpptest: clean $(MAKE) all CC=g++ CFLAGS="-O3 -I../lib -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" +c_standards: clean + $(MAKE) all CFLAGS="-std=gnu90" + $(MAKE) clean + $(MAKE) all CFLAGS="-std=c99" + $(MAKE) clean + $(MAKE) all CFLAGS="-std=gnu99" + $(MAKE) clean + $(MAKE) all CFLAGS="-std=c11" + $(MAKE) clean + clangtest: clean - CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) all CC=clang + $(MAKE) all CC=clang CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" sanitize: clean - CFLAGS="-O3 -g -fsanitize=undefined" $(MAKE) test CC=clang FUZZER_TIME="-T1mn" NB_LOOPS=-i1 + $(MAKE) test CC=clang FUZZER_TIME="-T1mn" NB_LOOPS=-i1 CFLAGS="-O3 -g -fsanitize=undefined" staticAnalyze: clean - CFLAGS=-g scan-build --status-bugs -v $(MAKE) all + scan-build --status-bugs -v $(MAKE) all CFLAGS=-g armtest: clean - CFLAGS="-O3 -Werror" $(MAKE) -C $(LZ4DIR) all CC=arm-linux-gnueabi-gcc - CFLAGS="-O3 -Werror" $(MAKE) -C $(PRGDIR) bins CC=arm-linux-gnueabi-gcc - CFLAGS="-O3 -Werror" $(MAKE) -C $(TESTDIR) bins CC=arm-linux-gnueabi-gcc + $(MAKE) -C $(LZ4DIR) all CC=arm-linux-gnueabi-gcc CFLAGS="-O3 -Werror" + $(MAKE) -C $(PRGDIR) bins CC=arm-linux-gnueabi-gcc CFLAGS="-O3 -Werror" + $(MAKE) -C $(TESTDIR) bins CC=arm-linux-gnueabi-gcc CFLAGS="-O3 -Werror" versionsTest: clean $(MAKE) -C $(TESTDIR) $@ diff --git a/tests/Makefile b/tests/Makefile index 2da6408..d5aad60 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -122,9 +122,9 @@ versionsTest: #FreeBSD targets ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD)) -test: test-lz4 test-lz4c test-fasttest test-frametest test-fullbench test-fuzzer test-mem +test: test-lz4 test-lz4c test-fasttest test-frametest test-fullbench test-fuzzer -test32: test-lz4c32 test-frametest32 test-fullbench32 test-fuzzer32 test-mem32 +test32: test-lz4c32 test-frametest32 test-fullbench32 test-fuzzer32 test-all: test test32 -- cgit v0.12 From 9b3dcb325f13c4c37c25c114f0b5bca62e6e30de Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Mon, 7 Nov 2016 18:21:59 +0100 Subject: MOREFLAGS="-Werror" --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4a18b17..4d6bdc4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ matrix: env: Ubu=12.04cont Cmd="make gpptest && make clean && make examples && make clean && make cmake && make clean && make travis-install" COMPILER=cc - os: linux sudo: false - env: Ubu=12.04cont Cmd='make test && make clean' COMPILER=cc + env: Ubu=12.04cont Cmd='make test' COMPILER=cc - os: linux sudo: false env: Ubu=12.04cont Cmd='make clangtest' COMPILER=clang @@ -51,7 +51,7 @@ matrix: - python3 - libc6-dev-i386 - gcc-multilib - - env: Ubu=14.04 Cmd='make -C tests test-lz4 test-lz4c32 CC=gcc-5' COMPILER=gcc-5 + - env: Ubu=14.04 Cmd='make -C tests test-lz4 test-lz4c32 CC=gcc-5 MOREFLAGS="-Werror"' COMPILER=gcc-5 dist: trusty sudo: required addons: @@ -63,7 +63,7 @@ matrix: - gcc-multilib - gcc-5 - gcc-5-multilib - - env: Ubu=14.04 Cmd='make c_standards && make -C tests test-lz4 CC=gcc-6' COMPILER=gcc-6 + - env: Ubu=14.04 Cmd='make c_standards CC=gcc-6 && make -C tests test-lz4 CC=gcc-6 MOREFLAGS="-Werror"' COMPILER=gcc-6 dist: trusty sudo: required addons: -- cgit v0.12 From 00c6370ad2419ea1906cf876013fef44df9c5af0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 7 Nov 2016 12:33:10 -0800 Subject: Fix #243 : detect and report fread() errors, by @iyokan --- NEWS | 7 +++++-- programs/lz4io.c | 21 +++++++++++---------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/NEWS b/NEWS index 9cd080e..0276769 100644 --- a/NEWS +++ b/NEWS @@ -2,13 +2,16 @@ v1.7.2 Changed : moved to versioning; package, cli and library have same version number Improved: Small decompression speed boost Improved: Small compression speed improvement on 64-bits systems -Improved: Performance on ARMv6 and ARMv7 +Improved: Significant speed boost on ARMv6 and ARMv7 +New : cli : --rm command +New : cli : file attributes are preserved, by Przemyslaw Skibinki Added : Debianization, by Evgeniy Polyakov Makefile: Generates object files (*.o) for faster (re)compilation on low power systems -New : recursive mode in benchmark, by Przemyslaw Skibinski +New : recursive mode in benchmark Fix : cli : crash on some invalid inputs Fix : cli : -t correctly validates lz4-compressed files, by Nick Terrell Fix : better ratio on 64-bits big-endian targets +Fix : cli : detects and reports fread() errors, thanks to Hiroshi Fujishima report #243 r131 New : Dos/DJGPP target, thanks to Louis Santillan (#114) diff --git a/programs/lz4io.c b/programs/lz4io.c index 02f4c2b..73957e1 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -466,20 +466,21 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, /* read first block */ readSize = fread(srcBuffer, (size_t)1, blockSize, srcFile); + if (ferror(srcFile)) EXM_THROW(30, "Error reading %s ", srcFileName); filesize += readSize; /* single-block file */ if (readSize < blockSize) { /* Compress in single pass */ size_t const cSize = LZ4F_compressFrame(dstBuffer, dstBufferSize, srcBuffer, readSize, &prefs); - if (LZ4F_isError(cSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(cSize)); + if (LZ4F_isError(cSize)) EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize)); compressedfilesize = cSize; DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (unsigned)(filesize>>20), (double)compressedfilesize/(filesize+!filesize)*100); /* avoid division by zero */ /* Write Block */ { size_t const sizeCheck = fwrite(dstBuffer, 1, cSize, dstFile); - if (sizeCheck!=cSize) EXM_THROW(35, "Write error : cannot write compressed block"); + if (sizeCheck!=cSize) EXM_THROW(32, "Write error : cannot write compressed block"); } } else @@ -488,9 +489,9 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, { /* Write Archive Header */ size_t headerSize = LZ4F_compressBegin(ctx, dstBuffer, dstBufferSize, &prefs); - if (LZ4F_isError(headerSize)) EXM_THROW(32, "File header generation failed : %s", LZ4F_getErrorName(headerSize)); + if (LZ4F_isError(headerSize)) EXM_THROW(33, "File header generation failed : %s", LZ4F_getErrorName(headerSize)); { size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); - if (sizeCheck!=headerSize) EXM_THROW(33, "Write error : cannot write header"); } + if (sizeCheck!=headerSize) EXM_THROW(34, "Write error : cannot write header"); } compressedfilesize += headerSize; /* Main Loop */ @@ -499,26 +500,26 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, /* Compress Block */ outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL); - if (LZ4F_isError(outSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(outSize)); + if (LZ4F_isError(outSize)) EXM_THROW(35, "Compression failed : %s", LZ4F_getErrorName(outSize)); compressedfilesize += outSize; DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100); /* Write Block */ { size_t const sizeCheck = fwrite(dstBuffer, 1, outSize, dstFile); - if (sizeCheck!=outSize) EXM_THROW(35, "Write error : cannot write compressed block"); } + if (sizeCheck!=outSize) EXM_THROW(36, "Write error : cannot write compressed block"); } /* Read next block */ readSize = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile); filesize += readSize; } - if (ferror(srcFile)) EXM_THROW(36, "Error reading %s ", srcFileName); + if (ferror(srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName); /* End of Stream mark */ headerSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL); - if (LZ4F_isError(headerSize)) EXM_THROW(37, "End of file generation failed : %s", LZ4F_getErrorName(headerSize)); + if (LZ4F_isError(headerSize)) EXM_THROW(38, "End of file generation failed : %s", LZ4F_getErrorName(headerSize)); { size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); - if (sizeCheck!=headerSize) EXM_THROW(38, "Write error : cannot write end of stream"); } + if (sizeCheck!=headerSize) EXM_THROW(39, "Write error : cannot write end of stream"); } compressedfilesize += headerSize; } @@ -532,7 +533,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, UTIL_setFileStat(dstFileName, &statbuf); } - if (g_removeSrcFile) { if (remove(srcFileName)) EXM_THROW(39, "Remove error : %s: %s", srcFileName, strerror(errno)); } /* remove source file : --rm */ + if (g_removeSrcFile) { if (remove(srcFileName)) EXM_THROW(40, "Remove error : %s: %s", srcFileName, strerror(errno)); } /* remove source file : --rm */ /* Final Status */ DISPLAYLEVEL(2, "\r%79s\r", ""); -- cgit v0.12 From e9089bcbe453bfd97d9303fc14194ccd7a4b9d59 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 7 Nov 2016 17:31:38 -0800 Subject: small compression ratio and speed improvement on small files --- NEWS | 1 + lib/lz4.c | 29 ++++++++++++----------------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/NEWS b/NEWS index 0276769..c349991 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,7 @@ v1.7.2 Changed : moved to versioning; package, cli and library have same version number Improved: Small decompression speed boost Improved: Small compression speed improvement on 64-bits systems +Improved: Small compression ratio and speed improvement on small files Improved: Significant speed boost on ARMv6 and ARMv7 New : cli : --rm command New : cli : file attributes are preserved, by Przemyslaw Skibinki diff --git a/lib/lz4.c b/lib/lz4.c index 85999ae..53b503c 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -412,7 +412,7 @@ int LZ4_sizeofState() { return LZ4_STREAMSIZE; } /*-****************************** * Compression functions ********************************/ -static U32 LZ4_hashSequence(U32 sequence, tableType_t const tableType) +static U32 LZ4_hash4(U32 sequence, tableType_t const tableType) { if (tableType == byU16) return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); @@ -420,7 +420,7 @@ static U32 LZ4_hashSequence(U32 sequence, tableType_t const tableType) return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); } -static U32 LZ4_hashSequence64(U64 sequence, tableType_t const tableType) +static U32 LZ4_hash5(U64 sequence, tableType_t const tableType) { static const U64 prime5bytes = 889523592379ULL; static const U64 prime8bytes = 11400714785074694791ULL; @@ -431,15 +431,10 @@ static U32 LZ4_hashSequence64(U64 sequence, tableType_t const tableType) return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); } -static U32 LZ4_hashSequenceT(size_t sequence, tableType_t const tableType) +FORCE_INLINE U32 LZ4_hashPosition(const void* p, tableType_t const tableType) { - if (LZ4_64bits()) return LZ4_hashSequence64(sequence, tableType); - return LZ4_hashSequence((U32)sequence, tableType); -} - -static U32 LZ4_hashPosition(const void* p, tableType_t tableType) -{ - return LZ4_hashSequenceT(LZ4_read_ARCH(p), tableType); + if ((LZ4_64bits()) && (tableType == byU32)) return LZ4_hash5(LZ4_read_ARCH(p), tableType); + return LZ4_hash4(LZ4_read32(p), tableType); } static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) @@ -452,7 +447,7 @@ static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableTy } } -static void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) { U32 const h = LZ4_hashPosition(p, tableType); LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); @@ -465,7 +460,7 @@ static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tab { const U16* const hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ } -static const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) { U32 const h = LZ4_hashPosition(p, tableType); return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); @@ -655,7 +650,7 @@ _next_match: _last_literals: /* Encode Last Literals */ - { const size_t lastRun = (size_t)(iend - anchor); + { size_t const lastRun = (size_t)(iend - anchor); if ( (outputLimited) && /* Check output buffer overflow */ ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) ) return 0; @@ -683,12 +678,12 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int if (maxOutputSize >= LZ4_compressBound(inputSize)) { if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); else - return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); } else { if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); else return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); } @@ -704,7 +699,7 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp void* const ctxPtr = &ctx; #endif - int result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); + int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); #if (HEAPMODE) FREEMEM(ctxPtr); -- cgit v0.12 From c7ab95faa5aabec7e2c01e66ce164865d7c79bbf Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 7 Nov 2016 19:32:24 -0800 Subject: slightly improved HC compression ratio --- lib/lz4hc.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index f109622..edc2db0 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -332,11 +332,11 @@ FORCE_INLINE int LZ4HC_encodeSequence ( static int LZ4HC_compress_generic ( - void* ctxvoid, - const char* source, - char* dest, - int inputSize, - int maxOutputSize, + void* const ctxvoid, + const char* const source, + char* const dest, + int const inputSize, + int const maxOutputSize, int compressionLevel, limitedOutput_directive limit ) @@ -381,7 +381,7 @@ static int LZ4HC_compress_generic ( _Search2: if (ip+ml < mflimit) - ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2, maxNbAttempts); + ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2, maxNbAttempts); else ml2 = ml; if (ml2 == ml) { /* No better match */ -- cgit v0.12