summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.travis.yml4
-rw-r--r--Makefile10
-rw-r--r--circle.yml1
-rw-r--r--contrib/cmake_unofficial/CMakeLists.txt14
-rw-r--r--doc/lz4_manual.html2
-rw-r--r--lib/Makefile9
-rw-r--r--lib/lz4.c31
-rw-r--r--lib/lz4.h4
-rw-r--r--lib/lz4frame.c4
-rw-r--r--lib/lz4hc.c3
-rw-r--r--lib/lz4hc.h8
-rw-r--r--programs/Makefile2
-rw-r--r--programs/bench.c7
-rw-r--r--programs/lz4.1.md7
-rw-r--r--programs/lz4cli.c35
-rw-r--r--tests/.gitignore6
-rw-r--r--tests/Makefile16
-rw-r--r--tests/fullbench.c18
-rw-r--r--tests/fuzzer.c21
-rw-r--r--tests/roundTripTest.c248
20 files changed, 414 insertions, 36 deletions
diff --git a/.travis.yml b/.travis.yml
index 0a876f9..de6875b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -49,6 +49,10 @@ matrix:
packages:
- valgrind
+ - env: Ubu=14.04 Cmd='make ctocpptest' COMPILER=cc
+ dist: trusty
+ sudo: false
+
- env: Ubu=14.04 Cmd='make -C tests test-lz4c32 test-fullbench32 versionsTest' COMPILER=cc
dist: trusty
sudo: required
diff --git a/Makefile b/Makefile
index 86613fd..5776b16 100644
--- a/Makefile
+++ b/Makefile
@@ -89,7 +89,7 @@ clean:
#-----------------------------------------------------------------------------
# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
#-----------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku))
HOST_OS = POSIX
.PHONY: install uninstall
@@ -172,6 +172,14 @@ gpptest gpptest32: clean
CC=$(CC) $(MAKE) -C $(PRGDIR) all CFLAGS="$(CFLAGS)"
CC=$(CC) $(MAKE) -C $(TESTDIR) all CFLAGS="$(CFLAGS)"
+ctocpptest: LIBCC="$(CC)"
+ctocpptest: TESTCC="$(CXX)"
+ctocpptest: CFLAGS=""
+ctocpptest: clean
+ CC=$(LIBCC) $(MAKE) -C $(LZ4DIR) CFLAGS="$(CFLAGS)" all
+ CC=$(LIBCC) $(MAKE) -C $(TESTDIR) CFLAGS="$(CFLAGS)" lz4.o lz4hc.o lz4frame.o
+ CC=$(TESTCC) $(MAKE) -C $(TESTDIR) CFLAGS="$(CFLAGS)" all
+
c_standards: clean
CFLAGS="-std=c90 -Werror" $(MAKE) clean allmost
CFLAGS="-std=gnu90 -Werror" $(MAKE) clean allmost
diff --git a/circle.yml b/circle.yml
index fa37590..1602e49 100644
--- a/circle.yml
+++ b/circle.yml
@@ -11,6 +11,7 @@ test:
- clang -v; make clangtest && make clean
- g++ -v; make gpptest && make clean
- gcc -v; make c_standards && make clean
+ - gcc -v; g++ -v; make ctocpptest && make clean
- gcc-5 -v; CC=gcc-5 CFLAGS="-O2 -Werror" make check && make clean
- gcc-5 -v; CC=gcc-5 CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean
- gcc-6 -v; CC=gcc-6 make c_standards && make clean
diff --git a/contrib/cmake_unofficial/CMakeLists.txt b/contrib/cmake_unofficial/CMakeLists.txt
index 27c3a78..b09c4fb 100644
--- a/contrib/cmake_unofficial/CMakeLists.txt
+++ b/contrib/cmake_unofficial/CMakeLists.txt
@@ -12,6 +12,8 @@
set(LZ4_TOP_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
+option(LZ4_BUILD_LEGACY_LZ4C "Build lz4c progam with legacy argument support" ON)
+
# Parse version information
file(STRINGS "${LZ4_TOP_SOURCE_DIR}/lib/lz4.h" LZ4_VERSION_MAJOR REGEX "^#define LZ4_VERSION_MAJOR +([0-9]+) +.*$")
string(REGEX REPLACE "^#define LZ4_VERSION_MAJOR +([0-9]+) +.*$" "\\1" LZ4_VERSION_MAJOR "${LZ4_VERSION_MAJOR}")
@@ -122,14 +124,18 @@ else()
endif()
# lz4
+set(LZ4_PROGRAMS_BUILT lz4cli)
add_executable(lz4cli ${LZ4_CLI_SOURCES})
set_target_properties(lz4cli PROPERTIES OUTPUT_NAME lz4)
target_link_libraries(lz4cli ${LZ4_LINK_LIBRARY})
# lz4c
-add_executable(lz4c ${LZ4_CLI_SOURCES})
-set_target_properties(lz4c PROPERTIES COMPILE_DEFINITIONS "ENABLE_LZ4C_LEGACY_OPTIONS")
-target_link_libraries(lz4c ${LZ4_LINK_LIBRARY})
+if (LZ4_BUILD_LEGACY_LZ4C)
+ list(APPEND LZ4_PROGRAMS_BUILT lz4c)
+ add_executable(lz4c ${LZ4_CLI_SOURCES})
+ set_target_properties(lz4c PROPERTIES COMPILE_DEFINITIONS "ENABLE_LZ4C_LEGACY_OPTIONS")
+ target_link_libraries(lz4c ${LZ4_LINK_LIBRARY})
+endif()
# Extra warning flags
include (CheckCCompilerFlag)
@@ -165,7 +171,7 @@ endforeach (flag)
if(NOT LZ4_BUNDLED_MODE)
include(GNUInstallDirs)
- install(TARGETS lz4cli lz4c
+ install(TARGETS ${LZ4_PROGRAMS_BUILT}
RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}")
install(TARGETS ${LZ4_LIBRARIES_BUILT}
LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index e5044fe..3fc71e4 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -214,7 +214,7 @@ int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
<pre><b>int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
</b><p> An LZ4_streamDecode_t context can be allocated once and re-used multiple times.
Use this function to start decompression of a new stream of blocks.
- A dictionary can optionnally be set. Use NULL or size 0 for a reset order.
+ A dictionary can optionally be set. Use NULL or size 0 for a reset order.
Dictionary is presumed stable : it must remain accessible and unmodified during next decompression.
@return : 1 if OK, 0 if error
diff --git a/lib/Makefile b/lib/Makefile
index abb6c07..6b37839 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -45,6 +45,7 @@ LIBVER := $(shell echo $(LIBVER_SCRIPT))
BUILD_SHARED:=yes
BUILD_STATIC:=yes
+OS ?= $(shell uname)
CPPFLAGS+= -DXXH_NAMESPACE=LZ4_
CFLAGS ?= -O3
DEBUGFLAGS:= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
@@ -58,7 +59,7 @@ SRCFILES := $(sort $(wildcard *.c))
# OS X linker doesn't support -soname, and use different extension
# see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html
-ifeq ($(shell uname), Darwin)
+ifeq ($(OS), Darwin)
SHARED_EXT = dylib
SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
@@ -123,7 +124,7 @@ clean:
#-----------------------------------------------------------------------------
# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
#-----------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku))
.PHONY: listL120
listL120: # extract lines >= 120 characters in *.{c,h}, by Takayuki Matsuoka (note : $$, for Makefile compatibility)
@@ -142,14 +143,14 @@ libdir ?= $(LIBDIR)
INCLUDEDIR ?= $(prefix)/include
includedir ?= $(INCLUDEDIR)
-ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly))
+ifneq (,$(filter $(OS),OpenBSD FreeBSD NetBSD DragonFly))
PKGCONFIGDIR ?= $(prefix)/libdata/pkgconfig
else
PKGCONFIGDIR ?= $(libdir)/pkgconfig
endif
pkgconfigdir ?= $(PKGCONFIGDIR)
-ifneq (,$(filter $(shell uname),SunOS))
+ifneq (,$(filter $(OS),SunOS))
INSTALL ?= ginstall
else
INSTALL ?= install
diff --git a/lib/lz4.c b/lib/lz4.c
index e51a3e0..05454fc 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -496,6 +496,21 @@ int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); }
int LZ4_sizeofState() { return LZ4_STREAMSIZE; }
+/*-************************************
+* Internal Definitions used in Tests
+**************************************/
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize);
+
+int LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize, const void* dict, size_t dictSize);
+
+#if defined (__cplusplus)
+}
+#endif
+
/*-******************************
* Compression functions
********************************/
@@ -699,7 +714,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
cctx->dictSize += (U32)inputSize;
}
cctx->currentOffset += (U32)inputSize;
- cctx->tableType = tableType;
+ cctx->tableType = (U16)tableType;
if (inputSize<LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */
@@ -1645,10 +1660,10 @@ static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, i
(BYTE*)dest-prefixSize, NULL, 0);
}
-LZ4_FORCE_O2_GCC_PPC64LE /* Exported under another name, for tests/fullbench.c */
-#define LZ4_decompress_safe_extDict LZ4_decompress_safe_forceExtDict
-int LZ4_decompress_safe_extDict(const char* source, char* dest, int compressedSize, int maxOutputSize,
- const void* dictStart, size_t dictSize)
+LZ4_FORCE_O2_GCC_PPC64LE
+int LZ4_decompress_safe_forceExtDict(const char* source, char* dest,
+ int compressedSize, int maxOutputSize,
+ const void* dictStart, size_t dictSize)
{
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
endOnInputSize, full, 0, usingExtDict,
@@ -1773,8 +1788,8 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch
/* The buffer wraps around, or they're switching to another buffer. */
lz4sd->extDictSize = lz4sd->prefixSize;
lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize;
- result = LZ4_decompress_safe_extDict(source, dest, compressedSize, maxOutputSize,
- lz4sd->externalDict, lz4sd->extDictSize);
+ result = LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize,
+ lz4sd->externalDict, lz4sd->extDictSize);
if (result <= 0) return result;
lz4sd->prefixSize = result;
lz4sd->prefixEnd = (BYTE*)dest + result;
@@ -1834,7 +1849,7 @@ int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressed
return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize);
return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, dictSize);
}
- return LZ4_decompress_safe_extDict(source, dest, compressedSize, maxOutputSize, dictStart, dictSize);
+ return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, dictSize);
}
int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize)
diff --git a/lib/lz4.h b/lib/lz4.h
index 7d13122..a0eddce 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -266,7 +266,7 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in
* 'dst' buffer must be already allocated.
* If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
*
- * Important : The previous 64KB of compressed data is assumed to remain present and unmodified in memory!
+ * Important : The previous 64KB of source data is assumed to remain present and unmodified in memory!
*
* Special 1 : When input is a double-buffer, they can have any size, including < 64 KB.
* Make sure that buffers are separated by at least one byte.
@@ -305,7 +305,7 @@ LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_str
/*! LZ4_setStreamDecode() :
* An LZ4_streamDecode_t context can be allocated once and re-used multiple times.
* Use this function to start decompression of a new stream of blocks.
- * A dictionary can optionnally be set. Use NULL or size 0 for a reset order.
+ * A dictionary can optionally be set. Use NULL or size 0 for a reset order.
* Dictionary is presumed stable : it must remain accessible and unmodified during next decompression.
* @return : 1 if OK, 0 if error
*/
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index e1d0b1d..08bf0fa 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -738,7 +738,7 @@ static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize,
static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
{
- int const acceleration = (level < -1) ? -level : 1;
+ int const acceleration = (level < 0) ? -level + 1 : 1;
LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent);
if (cdict) {
return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
@@ -749,7 +749,7 @@ static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize
static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
{
- int const acceleration = (level < -1) ? -level : 1;
+ int const acceleration = (level < 0) ? -level + 1 : 1;
(void)cdict; /* init once at beginning of frame */
return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
}
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 8108ea0..e913ee7 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -327,6 +327,8 @@ LZ4HC_InsertAndGetWiderMatch (
if (lookBackLength==0) { /* no back possible */
size_t const maxML = MIN(currentSegmentLength, srcPatternLength);
if ((size_t)longest < maxML) {
+ assert(base + matchIndex < ip);
+ if (ip - (base+matchIndex) > MAX_DISTANCE) break;
assert(maxML < 2 GB);
longest = (int)maxML;
*matchpos = base + matchIndex; /* virtual pos, relative to ip, to retrieve offset */
@@ -450,6 +452,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
*op += length;
/* Encode Offset */
+ assert( (*ip - match) <= MAX_DISTANCE ); /* note : consider providing offset as a value, rather than as a pointer difference */
LZ4_writeLE16(*op, (U16)(*ip-match)); *op += 2;
/* Encode MatchLength */
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index bb5e073..970fa39 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -246,6 +246,10 @@ LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") LZ4LIB_API int LZ4_resetStr
#ifndef LZ4_HC_SLO_098092834
#define LZ4_HC_SLO_098092834
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
/*! LZ4_compress_HC_destSize() : v1.8.0 (experimental)
* Will try to compress as much data from `src` as possible
* that can fit into `targetDstSize` budget.
@@ -343,5 +347,9 @@ int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* ds
*/
LZ4LIB_API void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC_t *dictionary_stream);
+#if defined (__cplusplus)
+}
+#endif
+
#endif /* LZ4_HC_SLO_098092834 */
#endif /* LZ4_HC_STATIC_LINKING_ONLY */
diff --git a/programs/Makefile b/programs/Makefile
index 9cc2d94..98366ad 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -109,7 +109,7 @@ clean:
#-----------------------------------------------------------------------------
# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
#-----------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku))
unlz4: lz4
ln -s lz4$(EXT) unlz4$(EXT)
diff --git a/programs/bench.c b/programs/bench.c
index 770191c..11bf044 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -49,7 +49,10 @@
#include "lz4.h"
#define COMPRESSOR0 LZ4_compress_local
-static int LZ4_compress_local(const char* src, char* dst, int srcSize, int dstSize, int clevel) { (void)clevel; return LZ4_compress_default(src, dst, srcSize, dstSize); }
+static int LZ4_compress_local(const char* src, char* dst, int srcSize, int dstSize, int clevel) {
+ int const acceleration = (clevel < 0) ? -clevel + 1 : 1;
+ return LZ4_compress_fast(src, dst, srcSize, dstSize, acceleration);
+}
#include "lz4hc.h"
#define COMPRESSOR1 LZ4_compress_HC
#define DEFAULTCOMPRESSOR COMPRESSOR0
@@ -326,7 +329,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
{ U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
if (crcOrig!=crcCheck) {
size_t u;
- DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
+ DISPLAY("\n!!! WARNING !!! %17s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
for (u=0; u<srcSize; u++) {
if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u]) {
U32 segNb, bNb, pos;
diff --git a/programs/lz4.1.md b/programs/lz4.1.md
index a5168e9..d4eaf8a 100644
--- a/programs/lz4.1.md
+++ b/programs/lz4.1.md
@@ -156,6 +156,13 @@ only the latest one will be applied.
* `-BD`:
Block Dependency (improves compression ratio on small blocks)
+* `--fast[=#]`:
+ switch to ultra-fast compression levels.
+ If `=#` is not present, it defaults to `1`.
+ The higher the value, the faster the compression speed, at the cost of some compression ratio.
+ This setting overwrites compression level if one was set previously.
+ Similarly, if a compression level is set after `--fast`, it overrides it.
+
* `--[no-]frame-crc`:
Select frame checksum (default:enabled)
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index ba519b4..dc60b00 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -141,6 +141,7 @@ static int usage_advanced(const char* exeName)
DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n");
DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n");
DISPLAY( "--favor-decSpeed: compressed files decompress faster, but are less compressed \n");
+ DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %u)\n", 1);
DISPLAY( "Benchmark arguments : \n");
DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n");
DISPLAY( " -e# : test all compression levels from -bX to # (default : 1)\n");
@@ -272,13 +273,26 @@ static unsigned readU32FromChar(const char** stringPtr)
return result;
}
+/** longCommandWArg() :
+ * check if *stringPtr is the same as longCommand.
+ * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
+ * @return 0 and doesn't modify *stringPtr otherwise.
+ */
+static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
+{
+ size_t const comSize = strlen(longCommand);
+ int const result = !strncmp(*stringPtr, longCommand, comSize);
+ if (result) *stringPtr += comSize;
+ return result;
+}
+
typedef enum { om_auto, om_compress, om_decompress, om_test, om_bench } operationMode_e;
int main(int argc, const char** argv)
{
int i,
cLevel=1,
- cLevelLast=1,
+ cLevelLast=-10000,
legacy_format=0,
forceStdout=0,
main_pause=0,
@@ -363,6 +377,25 @@ int main(int argc, const char** argv)
if (!strcmp(argument, "--help")) { usage_advanced(exeName); goto _cleanup; }
if (!strcmp(argument, "--keep")) { LZ4IO_setRemoveSrcFile(0); continue; } /* keep source file (default) */
if (!strcmp(argument, "--rm")) { LZ4IO_setRemoveSrcFile(1); continue; }
+ if (longCommandWArg(&argument, "--fast")) {
+ /* Parse optional acceleration factor */
+ if (*argument == '=') {
+ U32 fastLevel;
+ ++argument;
+ fastLevel = readU32FromChar(&argument);
+ if (fastLevel) {
+ cLevel = -(int)fastLevel;
+ } else {
+ badusage(exeName);
+ }
+ } else if (*argument != 0) {
+ /* Invalid character following --fast */
+ badusage(exeName);
+ } else {
+ cLevel = -1; /* default for --fast */
+ }
+ continue;
+ }
}
while (argument[1]!=0) {
diff --git a/tests/.gitignore b/tests/.gitignore
index 36dff42..9aa42a0 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -1,5 +1,5 @@
-# test build artefacts
+# build artefacts
datagen
frametest
frametest32
@@ -8,8 +8,12 @@ fullbench32
fuzzer
fuzzer32
fasttest
+roundTripTest
checkTag
# test artefacts
tmp*
versionsTest
+
+# local tests
+afl
diff --git a/tests/Makefile b/tests/Makefile
index d238561..bc43234 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -63,7 +63,7 @@ NB_LOOPS ?= -i1
default: all
-all: fullbench fuzzer frametest datagen
+all: fullbench fuzzer frametest roundTripTest datagen
all32: CFLAGS+=-m32
all32: all
@@ -103,6 +103,9 @@ fuzzer : lz4.o lz4hc.o xxhash.o fuzzer.c
frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
+roundTripTest : lz4.o lz4hc.o xxhash.o roundTripTest.c
+ $(CC) $(FLAGS) $^ -o $@$(EXT)
+
datagen : $(PRGDIR)/datagen.c datagencli.c
$(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT)
@@ -129,7 +132,7 @@ checkTag: checkTag.c $(LZ4DIR)/lz4.h
#-----------------------------------------------------------------------------
# validated only for Linux, OSX, BSD, Hurd and Solaris targets
#-----------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku))
MD5:=md5sum
ifneq (,$(filter $(shell uname), Darwin ))
@@ -262,8 +265,17 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat
cat tmp-tlb-hw >> tmp-tlb-hw.lz4
$(LZ4) -f tmp-tlb-hw.lz4 # uncompress valid frame followed by invalid data
$(LZ4) -BX tmp-tlb-hw -c -q | $(LZ4) -tv # test block checksum
+ # ./datagen -g20KB generates the same file every single time
+ # cannot save output of ./datagen -g20KB as input file to lz4 because the following shell commands are run before ./datagen -g20KB
+ test "$(shell ./datagen -g20KB | $(LZ4) -c --fast | wc -c)" -lt "$(shell ./datagen -g20KB | $(LZ4) -c --fast=9 | wc -c)" # -1 vs -9
+ test "$(shell ./datagen -g20KB | $(LZ4) -c -1 | wc -c)" -lt "$(shell ./datagen -g20KB| $(LZ4) -c --fast=1 | wc -c)" # 1 vs -1
+ test "$(shell ./datagen -g20KB | $(LZ4) -c --fast=1 | wc -c)" -eq "$(shell ./datagen -g20KB| $(LZ4) -c --fast| wc -c)" # checks default fast compression is -1
+ ! $(LZ4) -c --fast=0 tmp-tlb-dg20K # lz4 should fail when fast=0
+ ! $(LZ4) -c --fast=-1 tmp-tlb-dg20K # lz4 should fail when fast=-1
@$(RM) tmp-tlb*
+
+
test-lz4-dict: lz4 datagen
@echo "\n ---- test lz4 compression/decompression with dictionary ----"
./datagen -g16KB > tmp-dict
diff --git a/tests/fullbench.c b/tests/fullbench.c
index c06e230..2818ea2 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -220,8 +220,16 @@ static int local_LZ4_compress_fast_continue0(const char* in, char* out, int inSi
}
#ifndef LZ4_DLL_IMPORT
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
/* declare hidden function */
-int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize);
+extern int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize);
+
+#if defined (__cplusplus)
+}
+#endif
static int local_LZ4_compress_forceDict(const char* in, char* out, int inSize)
{
@@ -289,8 +297,16 @@ static int local_LZ4_decompress_safe_usingDict(const char* in, char* out, int in
}
#ifndef LZ4_DLL_IMPORT
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
extern int LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize, const void* dict, size_t dictSize);
+#if defined (__cplusplus)
+}
+#endif
+
static int local_LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize)
{
(void)inSize;
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 5dd75b3..6c79515 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -38,7 +38,7 @@
/*-************************************
* Dependencies
**************************************/
-#ifdef __unix__ /* must be included before platform.h for MAP_ANONYMOUS */
+#if defined(__unix__) && !defined(_AIX) /* must be included before platform.h for MAP_ANONYMOUS */
# include <sys/mman.h> /* mmap */
#endif
#include "platform.h" /* _CRT_SECURE_NO_WARNINGS */
@@ -48,6 +48,10 @@
#include <string.h> /* strcmp */
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
#include <assert.h>
+#if defined(__unix__) && defined(_AIX)
+# include <sys/mman.h> /* mmap */
+#endif
+
#define LZ4_STATIC_LINKING_ONLY
#define LZ4_HC_STATIC_LINKING_ONLY
#include "lz4hc.h"
@@ -319,12 +323,17 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
int result = 0;
unsigned cycleNb;
-# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \
- printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; }
+# define FUZ_CHECKTEST(cond, ...) \
+ if (cond) { \
+ printf("Test %u : ", testNb); printf(__VA_ARGS__); \
+ printf(" (seed %u, cycle %u) \n", seed, cycleNb); \
+ goto _output_error; \
+ }
+
# define FUZ_DISPLAYTEST(...) { \
testNb++; \
if (g_displayLevel>=4) { \
- printf("\r%4u - %2u ", cycleNb, testNb); \
+ printf("\r%4u - %2u :", cycleNb, testNb); \
printf(" " __VA_ARGS__); \
printf(" "); \
fflush(stdout); \
@@ -801,7 +810,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
- FUZ_DISPLAYTEST();
+ FUZ_DISPLAYTEST("LZ4_decompress_safe_usingDict with a too small output buffer");
{ U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
if ((U32)blockSize > missingBytes) {
decodedBuffer[blockSize-missingBytes] = 0;
@@ -811,7 +820,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
} }
/* Compress HC using External dictionary */
- FUZ_DISPLAYTEST();
+ FUZ_DISPLAYTEST("LZ4_compress_HC_continue with an external dictionary");
dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
LZ4_resetStreamHC (&LZ4dictHC, compressionLevel);
diff --git a/tests/roundTripTest.c b/tests/roundTripTest.c
new file mode 100644
index 0000000..2d34451
--- /dev/null
+++ b/tests/roundTripTest.c
@@ -0,0 +1,248 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/*
+ * This program takes a file in input,
+ * performs an LZ4 round-trip test (compress + decompress)
+ * compares the result with original
+ * and generates an abort() on corruption detection,
+ * in order for afl to register the event as a crash.
+*/
+
+
+/*===========================================
+* Tuning Constant
+*==========================================*/
+#ifndef MIN_CLEVEL
+# define MIN_CLEVEL (int)(-5)
+#endif
+
+
+
+/*===========================================
+* Dependencies
+*==========================================*/
+#include <stddef.h> /* size_t */
+#include <stdlib.h> /* malloc, free, exit */
+#include <stdio.h> /* fprintf */
+#include <string.h> /* strcmp */
+#include <assert.h>
+#include <sys/types.h> /* stat */
+#include <sys/stat.h> /* stat */
+#include "xxhash.h"
+
+#include "lz4.h"
+#include "lz4hc.h"
+
+
+/*===========================================
+* Macros
+*==========================================*/
+#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
+
+#define MSG(...) fprintf(stderr, __VA_ARGS__)
+
+#define CONTROL_MSG(c, ...) { \
+ if ((c)) { \
+ MSG(__VA_ARGS__); \
+ MSG(" \n"); \
+ abort(); \
+ } \
+}
+
+
+static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize)
+{
+ const char* const ip1 = (const char*)buff1;
+ const char* const ip2 = (const char*)buff2;
+ size_t pos;
+
+ for (pos=0; pos<buffSize; pos++)
+ if (ip1[pos]!=ip2[pos])
+ break;
+
+ return pos;
+}
+
+
+/* select a compression level
+ * based on first bytes present in a reference buffer */
+static int select_clevel(const void* refBuff, size_t refBuffSize)
+{
+ const int minCLevel = MIN_CLEVEL;
+ const int maxClevel = LZ4HC_CLEVEL_MAX;
+ const int cLevelSpan = maxClevel - minCLevel;
+ size_t const hashLength = MIN(16, refBuffSize);
+ unsigned const h32 = XXH32(refBuff, hashLength, 0);
+ int const randL = h32 % (cLevelSpan+1);
+
+ return minCLevel + randL;
+}
+
+
+typedef int (*compressFn)(const char* src, char* dst, int srcSize, int dstSize, int cLevel);
+
+
+/** roundTripTest() :
+ * Compresses `srcBuff` into `compressedBuff`,
+ * then decompresses `compressedBuff` into `resultBuff`.
+ * If clevel==0, compression level is derived from srcBuff's content head bytes.
+ * This function abort() if it detects any round-trip error.
+ * Therefore, if it returns, round trip is considered successfully validated.
+ * Note : `compressedBuffCapacity` should be `>= LZ4_compressBound(srcSize)`
+ * for compression to be guaranteed to work */
+static void roundTripTest(void* resultBuff, size_t resultBuffCapacity,
+ void* compressedBuff, size_t compressedBuffCapacity,
+ const void* srcBuff, size_t srcSize,
+ int clevel)
+{
+ int const proposed_clevel = clevel ? clevel : select_clevel(srcBuff, srcSize);
+ int const selected_clevel = proposed_clevel < 0 ? -proposed_clevel : proposed_clevel; /* if level < 0, it becomes an accelearion value */
+ compressFn compress = selected_clevel >= LZ4HC_CLEVEL_MIN ? LZ4_compress_HC : LZ4_compress_fast;
+ int const cSize = compress((const char*)srcBuff, (char*)compressedBuff, (int)srcSize, (int)compressedBuffCapacity, selected_clevel);
+ CONTROL_MSG(cSize == 0, "Compression error !");
+
+ { int const dSize = LZ4_decompress_safe((const char*)compressedBuff, (char*)resultBuff, cSize, (int)resultBuffCapacity);
+ CONTROL_MSG(dSize < 0, "Decompression detected an error !");
+ CONTROL_MSG(dSize != (int)srcSize, "Decompression corruption error : wrong decompressed size !");
+ }
+
+ /* check potential content corruption error */
+ assert(resultBuffCapacity >= srcSize);
+ { size_t const errorPos = checkBuffers(srcBuff, resultBuff, srcSize);
+ CONTROL_MSG(errorPos != srcSize,
+ "Silent decoding corruption, at pos %u !!!",
+ (unsigned)errorPos);
+ }
+
+}
+
+static void roundTripCheck(const void* srcBuff, size_t srcSize, int clevel)
+{
+ size_t const cBuffSize = LZ4_compressBound((int)srcSize);
+ void* const cBuff = malloc(cBuffSize);
+ void* const rBuff = malloc(cBuffSize);
+
+ if (!cBuff || !rBuff) {
+ fprintf(stderr, "not enough memory ! \n");
+ exit(1);
+ }
+
+ roundTripTest(rBuff, cBuffSize,
+ cBuff, cBuffSize,
+ srcBuff, srcSize,
+ clevel);
+
+ free(rBuff);
+ free(cBuff);
+}
+
+
+static size_t getFileSize(const char* infilename)
+{
+ int r;
+#if defined(_MSC_VER)
+ struct _stat64 statbuf;
+ r = _stat64(infilename, &statbuf);
+ if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */
+#else
+ struct stat statbuf;
+ r = stat(infilename, &statbuf);
+ if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
+#endif
+ return (size_t)statbuf.st_size;
+}
+
+
+static int isDirectory(const char* infilename)
+{
+ int r;
+#if defined(_MSC_VER)
+ struct _stat64 statbuf;
+ r = _stat64(infilename, &statbuf);
+ if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
+#else
+ struct stat statbuf;
+ r = stat(infilename, &statbuf);
+ if (!r && S_ISDIR(statbuf.st_mode)) return 1;
+#endif
+ return 0;
+}
+
+
+/** loadFile() :
+ * requirement : `buffer` size >= `fileSize` */
+static void loadFile(void* buffer, const char* fileName, size_t fileSize)
+{
+ FILE* const f = fopen(fileName, "rb");
+ if (isDirectory(fileName)) {
+ MSG("Ignoring %s directory \n", fileName);
+ exit(2);
+ }
+ if (f==NULL) {
+ MSG("Impossible to open %s \n", fileName);
+ exit(3);
+ }
+ { size_t const readSize = fread(buffer, 1, fileSize, f);
+ if (readSize != fileSize) {
+ MSG("Error reading %s \n", fileName);
+ exit(5);
+ } }
+ fclose(f);
+}
+
+
+static void fileCheck(const char* fileName, int clevel)
+{
+ size_t const fileSize = getFileSize(fileName);
+ void* const buffer = malloc(fileSize + !fileSize /* avoid 0 */);
+ if (!buffer) {
+ MSG("not enough memory \n");
+ exit(4);
+ }
+ loadFile(buffer, fileName, fileSize);
+ roundTripCheck(buffer, fileSize, clevel);
+ free (buffer);
+}
+
+
+int bad_usage(const char* exeName)
+{
+ MSG(" \n");
+ MSG("bad usage: \n");
+ MSG(" \n");
+ MSG("%s [Options] fileName \n", exeName);
+ MSG(" \n");
+ MSG("Options: \n");
+ MSG("-# : use #=[0-9] compression level (default:0 == random) \n");
+ return 1;
+}
+
+
+int main(int argCount, const char** argv)
+{
+ const char* const exeName = argv[0];
+ int argNb = 1;
+ int clevel = 0;
+
+ assert(argCount >= 1);
+ if (argCount < 2) return bad_usage(exeName);
+
+ if (argv[1][0] == '-') {
+ clevel = argv[1][1] - '0';
+ argNb = 2;
+ }
+
+ if (argNb >= argCount) return bad_usage(exeName);
+
+ fileCheck(argv[argNb], clevel);
+ MSG("no pb detected \n");
+ return 0;
+}
generated by cgit v0.12 at 2024-07-01 03:55:15 (GMT)