summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.travis.yml2
-rw-r--r--Makefile21
-rw-r--r--Makefile.inc87
-rw-r--r--NEWS7
-rw-r--r--README.md2
-rw-r--r--doc/lz4_manual.html33
-rw-r--r--doc/lz4frame_manual.html4
-rw-r--r--examples/Makefile11
-rw-r--r--lib/Makefile95
-rw-r--r--lib/liblz4-dll.rc.in35
-rw-r--r--lib/lz4.c65
-rw-r--r--lib/lz4.h25
-rw-r--r--lib/lz4frame.c8
-rw-r--r--lib/lz4frame.h4
-rw-r--r--lib/lz4hc.c3
-rw-r--r--programs/Makefile65
-rw-r--r--programs/bench.c12
-rw-r--r--programs/lz4-exe.rc.in27
-rw-r--r--programs/lz4.124
-rw-r--r--programs/lz4.1.md32
-rw-r--r--programs/lz4cli.c47
-rw-r--r--programs/lz4io.c147
-rw-r--r--programs/lz4io.h5
-rw-r--r--programs/util.h19
-rw-r--r--tests/Makefile59
-rw-r--r--tests/checkFrame.c2
-rw-r--r--tests/frametest.c21
-rw-r--r--tests/fullbench.c4
-rw-r--r--tests/fuzzer.c423
29 files changed, 825 insertions, 464 deletions
diff --git a/.travis.yml b/.travis.yml
index 301d294..2065478 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,7 +8,7 @@ matrix:
os: osx
compiler: clang
script:
- - make -C tests test-lz4 MOREFLAGS='-Werror -Wconversion -Wno-sign-conversion'
+ - make -C tests test-lz4 MOREFLAGS='-Werror -Wconversion -Wno-sign-conversion' | tee # test scenario where `stdout` is not the console
- CFLAGS=-m32 make -C tests clean test-lz4-contentSize
# Container-based 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
diff --git a/Makefile b/Makefile
index f3c6ce2..dd731eb 100644
--- a/Makefile
+++ b/Makefile
@@ -35,16 +35,7 @@ PRGDIR = programs
TESTDIR = tests
EXDIR = examples
-
-# Define nul output
-ifneq (,$(filter Windows%,$(OS)))
-EXT = .exe
-VOID = nul
-else
-EXT =
-VOID = /dev/null
-endif
-
+include Makefile.inc
.PHONY: default
default: lib-release lz4-release
@@ -93,7 +84,7 @@ clean:
#-----------------------------------------------------------------------------
# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
#-----------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku MidnightBSD MINGW32_NT-6.1 MINGW64_NT-6.1 MINGW32_NT-10.0 MINGW64_NT-10.0))
+ifeq ($(POSIX_ENV),Yes)
HOST_OS = POSIX
.PHONY: install uninstall
@@ -181,6 +172,14 @@ gpptest gpptest32: clean
CC=$(CC) $(MAKE) -C $(PRGDIR) all CFLAGS="$(CFLAGS)"
CC=$(CC) $(MAKE) -C $(TESTDIR) all CFLAGS="$(CFLAGS)"
+cxx17build : CC = "$(CXX) -Wno-deprecated"
+cxx17build : CFLAGS = -std=c++17 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror -pedantic
+cxx17build : clean
+ $(CXX) -v
+ CC=$(CC) $(MAKE) -C $(LZ4DIR) all CFLAGS="$(CFLAGS)"
+ CC=$(CC) $(MAKE) -C $(PRGDIR) all CFLAGS="$(CFLAGS)"
+ CC=$(CC) $(MAKE) -C $(TESTDIR) all CFLAGS="$(CFLAGS)"
+
ctocpptest: LIBCC="$(CC)"
ctocpptest: TESTCC="$(CXX)"
ctocpptest: CFLAGS=""
diff --git a/Makefile.inc b/Makefile.inc
new file mode 100644
index 0000000..2d64405
--- /dev/null
+++ b/Makefile.inc
@@ -0,0 +1,87 @@
+ifeq ($(V), 1)
+Q =
+else
+Q = @
+endif
+
+TARGET_OS ?= $(shell uname)
+ifeq ($(TARGET_OS),)
+ TARGET_OS ?= $(OS)
+endif
+
+ifneq (,$(filter Windows%,$(TARGET_OS)))
+LIBLZ4 = liblz4-$(LIBVER_MAJOR)
+LIBLZ4_EXP = liblz4.lib
+WINBASED = yes
+else
+LIBLZ4_EXP = liblz4.dll.a
+ ifneq (,$(filter MINGW%,$(TARGET_OS)))
+LIBLZ4 = liblz4
+WINBASED = yes
+ else
+ ifneq (,$(filter MSYS%,$(TARGET_OS)))
+LIBLZ4 = msys-lz4-$(LIBVER_MAJOR)
+WINBASED = yes
+ else
+ ifneq (,$(filter CYGWIN%,$(TARGET_OS)))
+LIBLZ4 = cyglz4-$(LIBVER_MAJOR)
+WINBASED = yes
+ else
+LIBLZ4 = liblz4.$(SHARED_EXT_VER)
+WINBASED = no
+EXT =
+ endif
+ endif
+ endif
+endif
+
+ifeq ($(WINBASED),yes)
+EXT = .exe
+WINDRES = windres
+endif
+
+#determine if dev/nul based on host environment
+ifneq (,$(filter MINGW% MSYS% CYGWIN%,$(shell uname)))
+VOID := /dev/null
+else
+ ifneq (,$(filter Windows%,$(OS)))
+VOID := nul
+ else
+VOID := /dev/null
+ endif
+endif
+
+ifneq (,$(filter Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku MidnightBSD MINGW% CYGWIN% MSYS%,$(shell uname)))
+POSIX_ENV = Yes
+else
+POSIX_ENV = No
+endif
+
+# Avoid symlinks when targetting Windows or building on a Windows host
+ifeq ($(WINBASED),yes)
+LN_S = cp -p
+LN_SF = cp -p
+else
+ ifneq (,$(filter MINGW% MSYS% CYGWIN%,$(shell uname)))
+LN_S = cp -p
+LN_SF = cp -p
+ else
+ ifneq (,$(filter Windows%,$(OS)))
+LN_S = cp -p
+LN_SF = cp -p
+ else
+LN_S = ln -s
+LN_SF = ln -sf
+ endif
+ endif
+endif
+
+ifneq (,$(filter $(shell uname),SunOS))
+INSTALL ?= ginstall
+else
+INSTALL ?= install
+endif
+
+INSTALL_PROGRAM ?= $(INSTALL) -m 755
+INSTALL_DATA ?= $(INSTALL) -m 644
+INSTALL_DIR ?= $(INSTALL) -d -m 755
diff --git a/NEWS b/NEWS
index 2d85607..860f15b 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,10 @@
+v1.9.1
+fix : decompression functions were reading a few bytes beyond input size (introduced in v1.9.0, reported by @ppodolsky and @danlark1)
+api : fix : lz4frame initializers compatibility with c++, reported by @degski
+cli : added command --list, based on a patch by @gabrielstedman
+build: improved Windows build, by @JPeterMugaas
+build: AIX, by Norman Green
+
v1.9.0
perf: large decompression speed improvement on x86/x64 (up to +20%) by @djwatson
api : changed : _destSize() compression variants are promoted to stable API
diff --git a/README.md b/README.md
index 73554cd..607fc4e 100644
--- a/README.md
+++ b/README.md
@@ -68,7 +68,7 @@ in single-thread mode.
| [Zstandard] 1.4.0 -1 | 2.883 | 515 MB/s | 1380 MB/s |
| LZF v3.6 | 2.073 | 415 MB/s | 910 MB/s |
| [zlib] deflate 1.2.11 -1| 2.730 | 100 MB/s | 415 MB/s |
-|**LZ4 HC -9 (v1.8.2)** |**2.721**| 41 MB/s | **4900 MB/s** |
+|**LZ4 HC -9 (v1.9.0)** |**2.721**| 41 MB/s | **4900 MB/s** |
| [zlib] deflate 1.2.11 -6| 3.099 | 36 MB/s | 445 MB/s |
[zlib]: http://www.zlib.net/
diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index 356a60d..3a9e0db 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -1,10 +1,10 @@
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
-<title>1.9.0 Manual</title>
+<title>1.9.1 Manual</title>
</head>
<body>
-<h1>1.9.0 Manual</h1>
+<h1>1.9.1 Manual</h1>
<hr>
<a name="Contents"></a><h2>Contents</h2>
<ol>
@@ -454,21 +454,18 @@ union LZ4_streamDecode_u {
</p></pre><BR>
-<pre><b>LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe() instead") LZ4LIB_API
-int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
-LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_continue() instead") LZ4LIB_API
-int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize);
-LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_usingDict() instead") LZ4LIB_API
-int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize);
-</b><p> These functions used to be a bit faster than LZ4_decompress_safe(),
- but situation has changed in recent versions.
- Now, `LZ4_decompress_safe()` is as fast and sometimes even faster than `LZ4_decompress_fast()`.
- Moreover, LZ4_decompress_safe() is protected vs malformed input, while `LZ4_decompress_fast()` is not, making it a security liability.
+<pre><b></b><p> These functions used to be faster than LZ4_decompress_safe(),
+ but it has changed, and they are now slower than LZ4_decompress_safe().
+ This is because LZ4_decompress_fast() doesn't know the input size,
+ and therefore must progress more cautiously in the input buffer to not read beyond the end of block.
+ On top of that `LZ4_decompress_fast()` is not protected vs malformed or malicious inputs, making it a security liability.
As a consequence, LZ4_decompress_fast() is strongly discouraged, and deprecated.
- Last LZ4_decompress_fast() specificity is that it can decompress a block without knowing its compressed size.
- Note that even that functionality could be achieved in a more secure manner if need be,
- though it would require new prototypes, and adaptation of the implementation to this new use case.
+ The last remaining LZ4_decompress_fast() specificity is that
+ it can decompress a block without knowing its compressed size.
+ Such functionality could be achieved in a more secure manner,
+ by also providing the maximum size of input buffer,
+ but it would require new prototypes, and adaptation of the implementation to this new use case.
Parameters:
originalSize : is the uncompressed size to regenerate.
@@ -477,9 +474,9 @@ int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize,
The function expects to finish at block's end exactly.
If the source stream is detected malformed, the function stops decoding and returns a negative result.
note : LZ4_decompress_fast*() requires originalSize. Thanks to this information, it never writes past the output buffer.
- However, since it doesn't know its 'src' size, it may read an unknown amount of input, and overflow input buffer.
- Also, since match offsets are not validated, match reads from 'src' may underflow.
- These issues never happen if input data is correct.
+ However, since it doesn't know its 'src' size, it may read an unknown amount of input, past input buffer bounds.
+ Also, since match offsets are not validated, match reads from 'src' may underflow too.
+ These issues never happen if input (compressed) data is correct.
But they may happen if input data is invalid (error or intentional tampering).
As a consequence, use these functions in trusted environments with trusted data **only**.
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index 914405f..2cf4f03 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -1,10 +1,10 @@
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
-<title>1.9.0 Manual</title>
+<title>1.9.1 Manual</title>
</head>
<body>
-<h1>1.9.0 Manual</h1>
+<h1>1.9.1 Manual</h1>
<hr>
<a name="Contents"></a><h2>Contents</h2>
<ol>
diff --git a/examples/Makefile b/examples/Makefile
index 103e7ec..6a34b33 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -36,16 +36,7 @@ TESTFILE = Makefile
LZ4DIR := ../lib
LZ4 = ../programs/lz4
-
-# Define *.exe as extension for Windows systems
-ifneq (,$(filter Windows%,$(OS)))
-EXT =.exe
-VOID = nul
-else
-EXT =
-VOID = /dev/null
-endif
-
+include ../Makefile.inc
default: all
diff --git a/lib/Makefile b/lib/Makefile
index cb1571c..330642a 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -45,7 +45,6 @@ LIBVER := $(shell echo $(LIBVER_SCRIPT))
BUILD_SHARED:=yes
BUILD_STATIC:=yes
-OS ?= $(shell uname)
CPPFLAGS+= -DXXH_NAMESPACE=LZ4_
CFLAGS ?= -O3
DEBUGFLAGS:= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
@@ -71,11 +70,7 @@ else
SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
endif
-ifneq (,$(filter Windows%,$(OS)))
-LIBLZ4 = liblz4-$(LIBVER_MAJOR)
-else
-LIBLZ4 = liblz4.$(SHARED_EXT_VER)
-endif
+include ../Makefile.inc
.PHONY: default
default: lib-release
@@ -90,12 +85,6 @@ all: lib
all32: CFLAGS+=-m32
all32: all
-ifeq ($(V), 1)
-Q =
-else
-Q = @
-endif
-
liblz4.a: $(SRCFILES)
ifeq ($(BUILD_STATIC),yes) # can be disabled on command line
@echo compiling static library
@@ -103,31 +92,50 @@ ifeq ($(BUILD_STATIC),yes) # can be disabled on command line
$(Q)$(AR) rcs $@ *.o
endif
+ifeq ($(WINBASED),yes)
+liblz4-dll.rc: liblz4-dll.rc.in
+ @echo creating library resource
+ $(Q)sed -e 's|@LIBLZ4@|$(LIBLZ4)|' \
+ -e 's|@LIBVER_MAJOR@|$(LIBVER_MAJOR)|g' \
+ -e 's|@LIBVER_MINOR@|$(LIBVER_MINOR)|g' \
+ -e 's|@LIBVER_PATCH@|$(LIBVER_PATCH)|g' \
+ $< >$@
+
+liblz4-dll.o: liblz4-dll.rc
+ $(WINDRES) -i liblz4-dll.rc -o liblz4-dll.o
+
+$(LIBLZ4): $(SRCFILES) liblz4-dll.o
+else
$(LIBLZ4): $(SRCFILES)
+endif
ifeq ($(BUILD_SHARED),yes) # can be disabled on command line
@echo compiling dynamic library $(LIBVER)
-ifneq (,$(filter Windows%,$(OS)))
- $(Q)$(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll/$@.dll -Wl,--out-implib,dll/liblz4.lib
-else
+ ifeq ($(WINBASED),yes)
+ $(Q)$(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll/$@.dll -Wl,--out-implib,dll/$(LIBLZ4_EXP)
+ else
$(Q)$(CC) $(FLAGS) -shared $^ -fPIC -fvisibility=hidden $(SONAME_FLAGS) -o $@
@echo creating versioned links
- $(Q)ln -sf $@ liblz4.$(SHARED_EXT_MAJOR)
- $(Q)ln -sf $@ liblz4.$(SHARED_EXT)
-endif
+ $(Q)$(LN_SF) $@ liblz4.$(SHARED_EXT_MAJOR)
+ $(Q)$(LN_SF) $@ liblz4.$(SHARED_EXT)
+ endif
endif
+ifeq (,$(filter MINGW%,$(TARGET_OS)))
liblz4: $(LIBLZ4)
+endif
clean:
- $(Q)$(RM) core *.o liblz4.pc dll/liblz4.dll dll/liblz4.lib
+ifeq ($(WINBASED),yes)
+ $(Q)$(RM) *.rc
+endif
+ $(Q)$(RM) core *.o liblz4.pc dll/$(LIBLZ4).dll dll/$(LIBLZ4_EXP)
$(Q)$(RM) *.a *.$(SHARED_EXT) *.$(SHARED_EXT_MAJOR) *.$(SHARED_EXT_VER)
@echo Cleaning library completed
-
#-----------------------------------------------------------------------------
# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
#-----------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku MidnightBSD MINGW32_NT-6.1 MINGW64_NT-6.1 MINGW32_NT-10.0 MINGW64_NT-10.0))
+ifeq ($(POSIX_ENV),Yes)
.PHONY: listL120
listL120: # extract lines >= 120 characters in *.{c,h}, by Takayuki Matsuoka (note : $$, for Makefile compatibility)
@@ -148,22 +156,13 @@ libdir ?= $(LIBDIR)
INCLUDEDIR ?= $(prefix)/include
includedir ?= $(INCLUDEDIR)
-ifneq (,$(filter $(OS),OpenBSD FreeBSD NetBSD DragonFly MidnightBSD))
+ ifneq (,$(filter $(TARGET_OS),OpenBSD FreeBSD NetBSD DragonFly MidnightBSD))
PKGCONFIGDIR ?= $(prefix)/libdata/pkgconfig
-else
+ else
PKGCONFIGDIR ?= $(libdir)/pkgconfig
-endif
+ endif
pkgconfigdir ?= $(PKGCONFIGDIR)
-ifneq (,$(filter $(OS),SunOS))
-INSTALL ?= ginstall
-else
-INSTALL ?= install
-endif
-
-INSTALL_PROGRAM ?= $(INSTALL)
-INSTALL_DATA ?= $(INSTALL) -m 644
-
liblz4.pc: liblz4.pc.in Makefile
@echo creating pkgconfig
$(Q)sed -e 's|@PREFIX@|$(prefix)|' \
@@ -173,26 +172,26 @@ liblz4.pc: liblz4.pc.in Makefile
$< >$@
install: lib liblz4.pc
- $(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(pkgconfigdir)/ $(DESTDIR)$(includedir)/ $(DESTDIR)$(libdir)/ $(DESTDIR)$(bindir)/
+ $(Q)$(INSTALL_DIR) $(DESTDIR)$(pkgconfigdir)/ $(DESTDIR)$(includedir)/ $(DESTDIR)$(libdir)/ $(DESTDIR)$(bindir)/
$(Q)$(INSTALL_DATA) liblz4.pc $(DESTDIR)$(pkgconfigdir)/
@echo Installing libraries
-ifeq ($(BUILD_STATIC),yes)
+ ifeq ($(BUILD_STATIC),yes)
$(Q)$(INSTALL_DATA) liblz4.a $(DESTDIR)$(libdir)/liblz4.a
$(Q)$(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(includedir)/lz4frame_static.h
-endif
-ifeq ($(BUILD_SHARED),yes)
+ endif
+ ifeq ($(BUILD_SHARED),yes)
# Traditionnally, one installs the DLLs in the bin directory as programs
# search them first in their directory. This allows to not pollute system
# directories (like c:/windows/system32), nor modify the PATH variable.
-ifneq (,$(filter Windows%,$(OS)))
+ ifeq ($(WINBASED),yes)
$(Q)$(INSTALL_PROGRAM) dll/$(LIBLZ4).dll $(DESTDIR)$(bindir)
- $(Q)$(INSTALL_PROGRAM) dll/liblz4.lib $(DESTDIR)$(libdir)
-else
+ $(Q)$(INSTALL_PROGRAM) dll/$(LIBLZ4_EXP) $(DESTDIR)$(libdir)
+ else
$(Q)$(INSTALL_PROGRAM) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)
- $(Q)ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_MAJOR)
- $(Q)ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT)
-endif
-endif
+ $(Q)$(LN_SF) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_MAJOR)
+ $(Q)$(LN_SF) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT)
+ endif
+ endif
@echo Installing headers in $(includedir)
$(Q)$(INSTALL_DATA) lz4.h $(DESTDIR)$(includedir)/lz4.h
$(Q)$(INSTALL_DATA) lz4hc.h $(DESTDIR)$(includedir)/lz4hc.h
@@ -201,14 +200,14 @@ endif
uninstall:
$(Q)$(RM) $(DESTDIR)$(pkgconfigdir)/liblz4.pc
-ifneq (,$(filter Windows%,$(OS)))
+ ifeq (WINBASED,1)
$(Q)$(RM) $(DESTDIR)$(bindir)/$(LIBLZ4).dll
- $(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.lib
-else
+ $(Q)$(RM) $(DESTDIR)$(libdir)/$(LIBLZ4_EXP)
+ else
$(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT)
$(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_MAJOR)
$(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_VER)
-endif
+ endif
$(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.a
$(Q)$(RM) $(DESTDIR)$(includedir)/lz4.h
$(Q)$(RM) $(DESTDIR)$(includedir)/lz4hc.h
diff --git a/lib/liblz4-dll.rc.in b/lib/liblz4-dll.rc.in
new file mode 100644
index 0000000..bf9adf5
--- /dev/null
+++ b/lib/liblz4-dll.rc.in
@@ -0,0 +1,35 @@
+#include <windows.h>
+
+// DLL version information.
+1 VERSIONINFO
+FILEVERSION @LIBVER_MAJOR@,@LIBVER_MINOR@,@LIBVER_PATCH@,0
+PRODUCTVERSION @LIBVER_MAJOR@,@LIBVER_MINOR@,@LIBVER_PATCH@,0
+FILEFLAGSMASK VS_FFI_FILEFLAGSMASK
+#ifdef _DEBUG
+ FILEFLAGS VS_FF_DEBUG | VS_FF_PRERELEASE
+#else
+ FILEFLAGS 0
+#endif
+FILEOS VOS_NT_WINDOWS32
+FILETYPE VFT_DLL
+FILESUBTYPE VFT2_UNKNOWN
+BEGIN
+ BLOCK "StringFileInfo"
+ BEGIN
+ BLOCK "040904B0"
+ BEGIN
+ VALUE "CompanyName", "Yann Collet"
+ VALUE "FileDescription", "Extremely fast compression"
+ VALUE "FileVersion", "@LIBVER_MAJOR@.@LIBVER_MINOR@.@LIBVER_PATCH@.0"
+ VALUE "InternalName", "@LIBLZ4@"
+ VALUE "LegalCopyright", "Copyright (C) 2013-2016, Yann Collet"
+ VALUE "OriginalFilename", "@LIBLZ4@.dll"
+ VALUE "ProductName", "LZ4"
+ VALUE "ProductVersion", "@LIBVER_MAJOR@.@LIBVER_MINOR@.@LIBVER_PATCH@.0"
+ END
+ END
+ BLOCK "VarFileInfo"
+ BEGIN
+ VALUE "Translation", 0x0409, 1200
+ END
+END
diff --git a/lib/lz4.c b/lib/lz4.c
index 693bdaf..e614c45 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -98,8 +98,14 @@
# define LZ4_SRC_INCLUDED 1
#endif
+#ifndef LZ4_STATIC_LINKING_ONLY
#define LZ4_STATIC_LINKING_ONLY
+#endif
+
+#ifndef LZ4_DISABLE_DEPRECATE_WARNINGS
#define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */
+#endif
+
#include "lz4.h"
/* see also "memory routines" below */
@@ -130,7 +136,7 @@
#endif /* LZ4_FORCE_INLINE */
/* LZ4_FORCE_O2_GCC_PPC64LE and LZ4_FORCE_O2_INLINE_GCC_PPC64LE
- * Gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy,
+ * gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy8,
* together with a simple 8-byte copy loop as a fall-back path.
* However, this optimization hurts the decompression speed by >30%,
* because the execution does not go to the optimized loop
@@ -138,10 +144,10 @@
* before going to the fall-back path become useless overhead.
* This optimization happens only with the -O3 flag, and -O2 generates
* a simple 8-byte copy loop.
- * With gcc on ppc64le, all of the LZ4_decompress_* and LZ4_wildCopy
+ * With gcc on ppc64le, all of the LZ4_decompress_* and LZ4_wildCopy8
* functions are annotated with __attribute__((optimize("O2"))),
- * and also LZ4_wildCopy is forcibly inlined, so that the O2 attribute
- * of LZ4_wildCopy does not affect the compression speed.
+ * and also LZ4_wildCopy8 is forcibly inlined, so that the O2 attribute
+ * of LZ4_wildCopy8 does not affect the compression speed.
*/
#if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) && !defined(__clang__)
# define LZ4_FORCE_O2_GCC_PPC64LE __attribute__((optimize("O2")))
@@ -295,7 +301,7 @@ static void LZ4_writeLE16(void* memPtr, U16 value)
/* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */
LZ4_FORCE_O2_INLINE_GCC_PPC64LE
-void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd)
+void LZ4_wildCopy8(void* dstPtr, const void* srcPtr, void* dstEnd)
{
BYTE* d = (BYTE*)dstPtr;
const BYTE* s = (const BYTE*)srcPtr;
@@ -336,7 +342,7 @@ LZ4_memcpy_using_offset_base(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, con
srcPtr += 8;
}
- LZ4_wildCopy(dstPtr, srcPtr, dstEnd);
+ LZ4_wildCopy8(dstPtr, srcPtr, dstEnd);
}
/* customized variant of memcpy, which can overwrite up to 32 bytes beyond dstEnd
@@ -940,7 +946,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
else *token = (BYTE)(litLength<<ML_BITS);
/* Copy Literals */
- LZ4_wildCopy(op, anchor, op+litLength);
+ LZ4_wildCopy8(op, anchor, op+litLength);
op+=litLength;
DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i",
(int)(anchor-(const BYTE*)source), litLength, (int)(ip-(const BYTE*)source));
@@ -1636,14 +1642,16 @@ LZ4_decompress_generic(
/* Currently the fast loop shows a regression on qualcomm arm chips. */
#if LZ4_FAST_DEC_LOOP
- if ((oend - op) < FASTLOOP_SAFE_DISTANCE)
+ if ((oend - op) < FASTLOOP_SAFE_DISTANCE) {
+ DEBUGLOG(6, "skip fast decode loop");
goto safe_decode;
+ }
/* Fast loop : decode sequences as long as output < iend-FASTLOOP_SAFE_DISTANCE */
while (1) {
/* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */
assert(oend - op >= FASTLOOP_SAFE_DISTANCE);
-
+ if (endOnInput) assert(ip < iend);
token = *ip++;
length = token >> ML_BITS; /* literal length */
@@ -1660,22 +1668,29 @@ LZ4_decompress_generic(
/* copy literals */
cpy = op+length;
LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH);
- if ( ((endOnInput) && ((cpy>oend-FASTLOOP_SAFE_DISTANCE) || (ip+length>iend-(2+1+LASTLITERALS))) )
- || ((!endOnInput) && (cpy>oend-FASTLOOP_SAFE_DISTANCE)) )
- {
- goto safe_literal_copy;
- }
- LZ4_wildCopy32(op, ip, cpy);
+ if (endOnInput) { /* LZ4_decompress_safe() */
+ if ((cpy>oend-32) || (ip+length>iend-32)) goto safe_literal_copy;
+ LZ4_wildCopy32(op, ip, cpy);
+ } else { /* LZ4_decompress_fast() */
+ if (cpy>oend-8) goto safe_literal_copy;
+ LZ4_wildCopy8(op, ip, cpy); /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time :
+ * it doesn't know input length, and only relies on end-of-block properties */
+ }
ip += length; op = cpy;
} else {
cpy = op+length;
- /* We don't need to check oend, since we check it once for each loop below */
- if ( ((endOnInput) && (ip+16>iend-(2+1+LASTLITERALS))))
- {
- goto safe_literal_copy;
- }
- /* Literals can only be 14, but hope compilers optimize if we copy by a register size */
- memcpy(op, ip, 16);
+ if (endOnInput) { /* LZ4_decompress_safe() */
+ DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length);
+ /* We don't need to check oend, since we check it once for each loop below */
+ if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) goto safe_literal_copy;
+ /* Literals can only be 14, but hope compilers optimize if we copy by a register size */
+ memcpy(op, ip, 16);
+ } else { /* LZ4_decompress_fast() */
+ /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time :
+ * it doesn't know input length, and relies on end-of-block properties */
+ memcpy(op, ip, 8);
+ if (length > 8) memcpy(op+8, ip+8, 8);
+ }
ip += length; op = cpy;
}
@@ -1838,7 +1853,7 @@ LZ4_decompress_generic(
}
} else {
- LZ4_wildCopy(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */
+ LZ4_wildCopy8(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */
ip += length; op = cpy;
}
@@ -1933,14 +1948,14 @@ LZ4_decompress_generic(
BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1);
if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */
if (op < oCopyLimit) {
- LZ4_wildCopy(op, match, oCopyLimit);
+ LZ4_wildCopy8(op, match, oCopyLimit);
match += oCopyLimit - op;
op = oCopyLimit;
}
while (op < cpy) *op++ = *match++;
} else {
memcpy(op, match, 8);
- if (length > 16) LZ4_wildCopy(op+8, match+8, cpy);
+ if (length > 16) LZ4_wildCopy8(op+8, match+8, cpy);
}
op = cpy; /* wildcopy correction */
}
diff --git a/lib/lz4.h b/lib/lz4.h
index 1589be9..a9c932c 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -97,7 +97,7 @@ extern "C" {
/*------ Version ------*/
#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */
#define LZ4_VERSION_MINOR 9 /* for new (non-breaking) interface capabilities */
-#define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */
+#define LZ4_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */
#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
@@ -631,15 +631,18 @@ LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4
LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
/*! LZ4_decompress_fast() : **unsafe!**
- * These functions are generally slightly faster than LZ4_decompress_safe(),
- * though the difference is small (generally ~5%).
- * However, the real cost is a risk : LZ4_decompress_safe() is protected vs malformed input, while `LZ4_decompress_fast()` is not, making it a security liability.
+ * These functions used to be faster than LZ4_decompress_safe(),
+ * but it has changed, and they are now slower than LZ4_decompress_safe().
+ * This is because LZ4_decompress_fast() doesn't know the input size,
+ * and therefore must progress more cautiously in the input buffer to not read beyond the end of block.
+ * On top of that `LZ4_decompress_fast()` is not protected vs malformed or malicious inputs, making it a security liability.
* As a consequence, LZ4_decompress_fast() is strongly discouraged, and deprecated.
- * These functions will generate a deprecation warning in the future.
*
- * Last LZ4_decompress_fast() specificity is that it can decompress a block without knowing its compressed size.
- * Note that even that functionality could be achieved in a more secure manner if need be,
- * though it would require new prototypes, and adaptation of the implementation to this new use case.
+ * The last remaining LZ4_decompress_fast() specificity is that
+ * it can decompress a block without knowing its compressed size.
+ * Such functionality could be achieved in a more secure manner,
+ * by also providing the maximum size of input buffer,
+ * but it would require new prototypes, and adaptation of the implementation to this new use case.
*
* Parameters:
* originalSize : is the uncompressed size to regenerate.
@@ -655,11 +658,11 @@ LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4
* As a consequence, use these functions in trusted environments with trusted data **only**.
*/
-/* LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe() instead") */
+LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe() instead")
LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
-/* LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_continue() instead") */
+LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_continue() instead")
LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize);
-/* LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_usingDict() instead") */
+LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_usingDict() instead")
LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize);
/*! LZ4_resetStream() :
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 42124e9..a10e4af 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -325,8 +325,7 @@ static size_t LZ4F_compressBound_internal(size_t srcSize,
const LZ4F_preferences_t* preferencesPtr,
size_t alreadyBuffered)
{
- LZ4F_preferences_t prefsNull;
- MEM_INIT(&prefsNull, 0, sizeof(prefsNull));
+ LZ4F_preferences_t prefsNull = LZ4F_INIT_PREFERENCES;
prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */
{ const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr;
U32 const flush = prefsPtr->autoFlush | (srcSize==0);
@@ -1065,7 +1064,10 @@ struct LZ4F_dctx_s {
LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber)
{
LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOC_AND_ZERO(sizeof(LZ4F_dctx));
- if (dctx==NULL) return err0r(LZ4F_ERROR_GENERIC);
+ if (dctx == NULL) { /* failed allocation */
+ *LZ4F_decompressionContextPtr = NULL;
+ return err0r(LZ4F_ERROR_allocation_failed);
+ }
dctx->version = versionNumber;
*LZ4F_decompressionContextPtr = dctx;
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index ca20dc9..742c252 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -176,7 +176,7 @@ typedef struct {
LZ4F_blockChecksum_t blockChecksumFlag; /* 1: each block followed by a checksum of block's compressed data; 0: disabled (default) */
} LZ4F_frameInfo_t;
-#define LZ4F_INIT_FRAMEINFO { 0, 0, 0, 0, 0, 0, 0 } /* v1.8.3+ */
+#define LZ4F_INIT_FRAMEINFO { LZ4F_default, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame, 0ULL, 0U, LZ4F_noBlockChecksum } /* v1.8.3+ */
/*! LZ4F_preferences_t :
* makes it possible to supply advanced compression instructions to streaming interface.
@@ -191,7 +191,7 @@ typedef struct {
unsigned reserved[3]; /* must be zero for forward compatibility */
} LZ4F_preferences_t;
-#define LZ4F_INIT_PREFERENCES { LZ4F_INIT_FRAMEINFO, 0, 0, 0, { 0, 0, 0 } } /* v1.8.3+ */
+#define LZ4F_INIT_PREFERENCES { LZ4F_INIT_FRAMEINFO, 0, 0u, 0u, { 0u, 0u, 0u } } /* v1.8.3+ */
/*-*********************************
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index d5f6743..936f739 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -442,7 +442,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
}
/* Copy Literals */
- LZ4_wildCopy(*op, *anchor, (*op) + length);
+ LZ4_wildCopy8(*op, *anchor, (*op) + length);
*op += length;
/* Encode Offset */
@@ -1396,6 +1396,7 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx,
} }
} /* for (cur = 1; cur <= last_match_pos; cur++) */
+ assert(last_match_pos < LZ4_OPT_NUM + TRAILING_LITERALS);
best_mlen = opt[last_match_pos].mlen;
best_off = opt[last_match_pos].off;
cur = last_match_pos - best_mlen;
diff --git a/programs/Makefile b/programs/Makefile
index 92fd683..4994551 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -56,17 +56,7 @@ LZ4_VERSION=$(LIBVER)
MD2ROFF = ronn
MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="lz4 $(LZ4_VERSION)"
-
-# Define *.exe as extension for Windows systems
-ifneq (,$(filter Windows%,$(OS)))
-EXT :=.exe
-VOID := nul
-else
-EXT :=
-VOID := /dev/null
-endif
-
-
+include ../Makefile.inc
default: lz4-release
@@ -75,14 +65,32 @@ all: lz4 lz4c
all32: CFLAGS+=-m32
all32: all
+ifeq ($(WINBASED),yes)
+lz4-exe.rc: lz4-exe.rc.in
+ @echo creating executable resource
+ $(Q)sed -e 's|@PROGNAME@|lz4|' \
+ -e 's|@LIBVER_MAJOR@|$(LIBVER_MAJOR)|g' \
+ -e 's|@LIBVER_MINOR@|$(LIBVER_MINOR)|g' \
+ -e 's|@LIBVER_PATCH@|$(LIBVER_PATCH)|g' \
+ -e 's|@EXT@|$(EXT)|g' \
+ $< >$@
+
+lz4-exe.o: lz4-exe.rc
+ $(WINDRES) -i lz4-exe.rc -o lz4-exe.o
+
+lz4: $(OBJFILES) lz4-exe.o
+ $(CC) $(FLAGS) $^ -o $@$(EXT)
+else
lz4: $(OBJFILES)
$(CC) $(FLAGS) $^ -o $@$(EXT)
+endif
+
lz4-release: DEBUGFLAGS=
lz4-release: lz4
lz4c: lz4
- ln -s lz4$(EXT) lz4c$(EXT)
+ $(LN_SF) lz4$(EXT) lz4c$(EXT)
lz4c32: CFLAGS += -m32
lz4c32 : $(SRCFILES)
@@ -100,6 +108,9 @@ preview-man: clean-man man
man ./lz4.1
clean:
+ifeq ($(WINBASED),yes)
+ $(Q)$(RM) *.rc
+endif
@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
@$(RM) core *.o *.test tmp* \
lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4$(EXT) lz4cat$(EXT)
@@ -109,13 +120,13 @@ clean:
#-----------------------------------------------------------------------------
# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
#-----------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku MidnightBSD MINGW32_NT-6.1 MINGW64_NT-6.1 MINGW32_NT-10.0 MINGW64_NT-10.0))
+ifeq ($(POSIX_ENV),Yes)
unlz4: lz4
- ln -s lz4$(EXT) unlz4$(EXT)
+ $(LN_SF) lz4$(EXT) unlz4$(EXT)
lz4cat: lz4
- ln -s lz4$(EXT) lz4cat$(EXT)
+ $(LN_SF) lz4$(EXT) lz4cat$(EXT)
DESTDIR ?=
# directory variables : GNU conventions prefer lowercase
@@ -134,28 +145,18 @@ mandir ?= $(MANDIR)
MAN1DIR ?= $(mandir)/man1
man1dir ?= $(MAN1DIR)
-ifneq (,$(filter $(shell uname),SunOS))
-INSTALL ?= ginstall
-else
-INSTALL ?= install
-endif
-
-INSTALL_PROGRAM ?= $(INSTALL) -m 755
-INSTALL_DATA ?= $(INSTALL) -m 644
-
-
install: lz4
@echo Installing binaries
- @$(INSTALL) -d -m 755 $(DESTDIR)$(bindir)/ $(DESTDIR)$(man1dir)/
+ @$(INSTALL_DIR) $(DESTDIR)$(bindir)/ $(DESTDIR)$(man1dir)/
@$(INSTALL_PROGRAM) lz4$(EXT) $(DESTDIR)$(bindir)/lz4$(EXT)
- @ln -sf lz4$(EXT) $(DESTDIR)$(bindir)/lz4c$(EXT)
- @ln -sf lz4$(EXT) $(DESTDIR)$(bindir)/lz4cat$(EXT)
- @ln -sf lz4$(EXT) $(DESTDIR)$(bindir)/unlz4$(EXT)
+ @$(LN_S) lz4$(EXT) $(DESTDIR)$(bindir)/lz4c$(EXT)
+ @$(LN_S) lz4$(EXT) $(DESTDIR)$(bindir)/lz4cat$(EXT)
+ @$(LN_S) lz4$(EXT) $(DESTDIR)$(bindir)/unlz4$(EXT)
@echo Installing man pages
@$(INSTALL_DATA) lz4.1 $(DESTDIR)$(man1dir)/lz4.1
- @ln -sf lz4.1 $(DESTDIR)$(man1dir)/lz4c.1
- @ln -sf lz4.1 $(DESTDIR)$(man1dir)/lz4cat.1
- @ln -sf lz4.1 $(DESTDIR)$(man1dir)/unlz4.1
+ @$(LN_SF) lz4.1 $(DESTDIR)$(man1dir)/lz4c.1
+ @$(LN_SF) lz4.1 $(DESTDIR)$(man1dir)/lz4cat.1
+ @$(LN_SF) lz4.1 $(DESTDIR)$(man1dir)/unlz4.1
@echo lz4 installation completed
uninstall:
diff --git a/programs/bench.c b/programs/bench.c
index 11bf044..5934935 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -209,7 +209,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
blockTable[nbBlocks].cPtr = cPtr;
blockTable[nbBlocks].resPtr = resPtr;
blockTable[nbBlocks].srcSize = thisBlockSize;
- blockTable[nbBlocks].cRoom = LZ4_compressBound((int)thisBlockSize);
+ blockTable[nbBlocks].cRoom = (size_t)LZ4_compressBound((int)thisBlockSize);
srcPtr += thisBlockSize;
cPtr += blockTable[nbBlocks].cRoom;
resPtr += thisBlockSize;
@@ -257,8 +257,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
for (nbLoops=0; nbLoops < nbCompressionLoops; nbLoops++) {
U32 blockNb;
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
- size_t const rSize = compP.compressionFunction(blockTable[blockNb].srcPtr, blockTable[blockNb].cPtr, (int)blockTable[blockNb].srcSize, (int)blockTable[blockNb].cRoom, cLevel);
- if (LZ4_isError(rSize)) EXM_THROW(1, "LZ4_compress() failed");
+ size_t const rSize = (size_t)compP.compressionFunction(blockTable[blockNb].srcPtr, blockTable[blockNb].cPtr, (int)blockTable[blockNb].srcSize, (int)blockTable[blockNb].cRoom, cLevel);
+ if (LZ4_isError(rSize)) EXM_THROW(1, "LZ4 compression failed");
blockTable[blockNb].cSize = rSize;
} }
{ U64 const clockSpan = UTIL_clockSpanNano(clockStart);
@@ -298,12 +298,12 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
for (nbLoops=0; nbLoops < nbDecodeLoops; nbLoops++) {
U32 blockNb;
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
- size_t const regenSize = LZ4_decompress_safe(blockTable[blockNb].cPtr, blockTable[blockNb].resPtr, (int)blockTable[blockNb].cSize, (int)blockTable[blockNb].srcSize);
- if (LZ4_isError(regenSize)) {
+ int const regenSize = LZ4_decompress_safe(blockTable[blockNb].cPtr, blockTable[blockNb].resPtr, (int)blockTable[blockNb].cSize, (int)blockTable[blockNb].srcSize);
+ if (regenSize < 0) {
DISPLAY("LZ4_decompress_safe() failed on block %u \n", blockNb);
break;
}
- blockTable[blockNb].resSize = regenSize;
+ blockTable[blockNb].resSize = (size_t)regenSize;
} }
{ U64 const clockSpan = UTIL_clockSpanNano(clockStart);
if (clockSpan > 0) {
diff --git a/programs/lz4-exe.rc.in b/programs/lz4-exe.rc.in
new file mode 100644
index 0000000..7b81030
--- /dev/null
+++ b/programs/lz4-exe.rc.in
@@ -0,0 +1,27 @@
+1 VERSIONINFO
+FILEVERSION @LIBVER_MAJOR@,@LIBVER_MINOR@,@LIBVER_PATCH@,0
+PRODUCTVERSION @LIBVER_MAJOR@,@LIBVER_MINOR@,@LIBVER_PATCH@,0
+FILEFLAGSMASK 0
+FILEOS 0x40000
+FILETYPE 1
+{
+ BLOCK "StringFileInfo"
+ {
+ BLOCK "040904B0"
+ {
+ VALUE "CompanyName", "Yann Collet"
+ VALUE "FileDescription", "Extremely fast compression"
+ VALUE "FileVersion", "@LIBVER_MAJOR@.@LIBVER_MINOR@.@LIBVER_PATCH@.0"
+ VALUE "InternalName", "@PROGNAME@"
+ VALUE "LegalCopyright", "Copyright (C) 2013-2016, Yann Collet"
+ VALUE "OriginalFilename", "@PROGNAME@.@EXT@"
+ VALUE "ProductName", "LZ4"
+ VALUE "ProductVersion", "@LIBVER_MAJOR@.@LIBVER_MINOR@.@LIBVER_PATCH@.0"
+ }
+ }
+ BLOCK "VarFileInfo"
+ {
+ VALUE "Translation", 0x0409, 1200
+ }
+}
+
diff --git a/programs/lz4.1 b/programs/lz4.1
index 1576e45..ad0c12c 100644
--- a/programs/lz4.1
+++ b/programs/lz4.1
@@ -1,5 +1,5 @@
.
-.TH "LZ4" "1" "April 2019" "lz4 1.9.0" "User Commands"
+.TH "LZ4" "1" "April 2019" "lz4 1.9.1" "User Commands"
.
.SH "NAME"
\fBlz4\fR \- lz4, unlz4, lz4cat \- Compress or decompress \.lz4 files
@@ -23,9 +23,6 @@ When writing scripts that need to decompress files, it is recommended to always
\fBlz4\fR supports a command line syntax similar \fIbut not identical\fR to \fBgzip(1)\fR\. Differences are :
.
.IP "\(bu" 4
-\fBlz4\fR preserves original files
-.
-.IP "\(bu" 4
\fBlz4\fR compresses a single file by default (see \fB\-m\fR for multiple files)
.
.IP "\(bu" 4
@@ -35,19 +32,16 @@ When writing scripts that need to decompress files, it is recommended to always
\fBlz4 file\.lz4\fR will default to decompression (use \fB\-z\fR to force compression)
.
.IP "\(bu" 4
-\fBlz4\fR shows real\-time notification statistics during compression or decompression of a single file (use \fB\-q\fR to silence them)
-.
-.IP "\(bu" 4
-If no destination name is provided, result is sent to \fBstdout\fR \fIexcept if stdout is the console\fR\.
+\fBlz4\fR preserves original files
.
.IP "\(bu" 4
-If no destination name is provided, \fBand\fR if \fBstdout\fR is the console, \fBfile\fR is compressed into \fBfile\.lz4\fR\.
+\fBlz4\fR shows real\-time notification statistics during compression or decompression of a single file (use \fB\-q\fR to silence them)
.
.IP "\(bu" 4
-As a consequence of previous rules, note the following example : \fBlz4 file | consumer\fR sends compressed data to \fBconsumer\fR through \fBstdout\fR, hence it does \fInot\fR create \fBfile\.lz4\fR\.
+When no destination is specified, result is sent on implicit output, which depends on \fBstdout\fR status\. When \fBstdout\fR \fIis Not the console\fR, it becomes the implicit output\. Otherwise, if \fBstdout\fR is the console, the implicit output is \fBfilename\.lz4\fR\.
.
.IP "\(bu" 4
-Another consequence of those rules is that to run \fBlz4\fR under \fBnohup\fR, you should provide a destination file: \fBnohup lz4 file file\.lz4\fR, because \fBnohup\fR writes the specified command\'s output to a file\.
+It is considered bad practice to rely on implicit output in scripts\. because the script\'s environment may change\. Always use explicit output in scripts\. \fB\-c\fR ensures that output will be \fBstdout\fR\. Conversely, providing a destination name, or using \fB\-m\fR ensures that the output will be either the specified name, or \fBfilename\.lz4\fR respectively\.
.
.IP "" 0
.
@@ -55,7 +49,7 @@ Another consequence of those rules is that to run \fBlz4\fR under \fBnohup\fR, y
Default behaviors can be modified by opt\-in commands, detailed below\.
.
.IP "\(bu" 4
-\fBlz4 \-m\fR makes it possible to provide multiple input filenames, which will be compressed into files using suffix \fB\.lz4\fR\. Progress notifications are also disabled by default (use \fB\-v\fR to enable them)\. This mode has a behavior which more closely mimics \fBgzip\fR command line, with the main remaining difference being that source files are preserved by default\.
+\fBlz4 \-m\fR makes it possible to provide multiple input filenames, which will be compressed into files using suffix \fB\.lz4\fR\. Progress notifications become disabled by default (use \fB\-v\fR to enable them)\. This mode has a behavior which more closely mimics \fBgzip\fR command line, with the main remaining difference being that source files are preserved by default\.
.
.IP "\(bu" 4
Similarly, \fBlz4 \-m \-d\fR can decompress multiple \fB*\.lz4\fR files\.
@@ -111,6 +105,10 @@ Test the integrity of compressed \fB\.lz4\fR files\. The decompressed data is di
\fB\-b#\fR
Benchmark mode, using \fB#\fR compression level\.
.
+.TP
+\fB\-\-list\fR
+List information about \.lz4 files\. note : current implementation is limited to single\-frame \.lz4 files\.
+.
.SS "Operation modifiers"
.
.TP
@@ -145,7 +143,7 @@ Force write to standard output, even if it is the console\.
.
.TP
\fB\-m\fR \fB\-\-multiple\fR
-Multiple input files\. Compressed file names will be appended a \fB\.lz4\fR suffix\. This mode also reduces notification level\. \fBlz4 \-m\fR has a behavior equivalent to \fBgzip \-k\fR (it preserves source files by default)\.
+Multiple input files\. Compressed file names will be appended a \fB\.lz4\fR suffix\. This mode also reduces notification level\. Can also be used to list multiple files\. \fBlz4 \-m\fR has a behavior equivalent to \fBgzip \-k\fR (it preserves source files by default)\.
.
.TP
\fB\-r\fR
diff --git a/programs/lz4.1.md b/programs/lz4.1.md
index 10449a0..8874467 100644
--- a/programs/lz4.1.md
+++ b/programs/lz4.1.md
@@ -31,29 +31,29 @@ The native file format is the `.lz4` format.
`lz4` supports a command line syntax similar _but not identical_ to `gzip(1)`.
Differences are :
- * `lz4` preserves original files
* `lz4` compresses a single file by default (see `-m` for multiple files)
* `lz4 file1 file2` means : compress file1 _into_ file2
* `lz4 file.lz4` will default to decompression (use `-z` to force compression)
+ * `lz4` preserves original files
* `lz4` shows real-time notification statistics
during compression or decompression of a single file
(use `-q` to silence them)
- * If no destination name is provided, result is sent to `stdout`
- _except if stdout is the console_.
- * If no destination name is provided, __and__ if `stdout` is the console,
- `file` is compressed into `file.lz4`.
- * As a consequence of previous rules, note the following example :
- `lz4 file | consumer` sends compressed data to `consumer` through `stdout`,
- hence it does _not_ create `file.lz4`.
- * Another consequence of those rules is that to run `lz4` under `nohup`,
- you should provide a destination file: `nohup lz4 file file.lz4`,
- because `nohup` writes the specified command's output to a file.
+ * When no destination is specified, result is sent on implicit output,
+ which depends on `stdout` status.
+ When `stdout` _is Not the console_, it becomes the implicit output.
+ Otherwise, if `stdout` is the console, the implicit output is `filename.lz4`.
+ * It is considered bad practice to rely on implicit output in scripts.
+ because the script's environment may change.
+ Always use explicit output in scripts.
+ `-c` ensures that output will be `stdout`.
+ Conversely, providing a destination name, or using `-m`
+ ensures that the output will be either the specified name, or `filename.lz4` respectively.
Default behaviors can be modified by opt-in commands, detailed below.
* `lz4 -m` makes it possible to provide multiple input filenames,
which will be compressed into files using suffix `.lz4`.
- Progress notifications are also disabled by default (use `-v` to enable them).
+ Progress notifications become disabled by default (use `-v` to enable them).
This mode has a behavior which more closely mimics `gzip` command line,
with the main remaining difference being that source files are preserved by default.
* Similarly, `lz4 -m -d` can decompress multiple `*.lz4` files.
@@ -81,8 +81,7 @@ In some cases, some options can be expressed using short command `-x`
or long command `--long-word`.
Short commands can be concatenated together.
For example, `-d -c` is equivalent to `-dc`.
-Long commands cannot be concatenated.
-They must be clearly separated by a space.
+Long commands cannot be concatenated. They must be clearly separated by a space.
### Multiple commands
@@ -114,6 +113,10 @@ only the latest one will be applied.
* `-b#`:
Benchmark mode, using `#` compression level.
+* `--list`:
+ List information about .lz4 files.
+ note : current implementation is limited to single-frame .lz4 files.
+
### Operation modifiers
* `-#`:
@@ -161,6 +164,7 @@ only the latest one will be applied.
Multiple input files.
Compressed file names will be appended a `.lz4` suffix.
This mode also reduces notification level.
+ Can also be used to list multiple files.
`lz4 -m` has a behavior equivalent to `gzip -k`
(it preserves source files by default).
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index 8bd7042..39ff1ea 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -141,6 +141,7 @@ static int usage_advanced(const char* exeName)
DISPLAY( " -BX : enable block checksum (default:disabled) \n");
DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled) \n");
DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n");
+ DISPLAY( "--list : lists information about .lz4 files. Useful if compressed with --content-size flag.\n");
DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n");
DISPLAY( "--favor-decSpeed: compressed files decompress faster, but are less compressed \n");
DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %i)\n", 1);
@@ -286,7 +287,7 @@ static int longCommandWArg(const char** stringPtr, const char* longCommand)
return result;
}
-typedef enum { om_auto, om_compress, om_decompress, om_test, om_bench } operationMode_e;
+typedef enum { om_auto, om_compress, om_decompress, om_test, om_bench, om_list } operationMode_e;
/** determineOpMode() :
* auto-determine operation mode, based on input filename extension
@@ -383,6 +384,7 @@ int main(int argc, const char** argv)
if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(prefs, 0); continue; }
if (!strcmp(argument, "--content-size")) { LZ4IO_setContentSize(prefs, 1); continue; }
if (!strcmp(argument, "--no-content-size")) { LZ4IO_setContentSize(prefs, 0); continue; }
+ if (!strcmp(argument, "--list")) { mode = om_list; continue; }
if (!strcmp(argument, "--sparse")) { LZ4IO_setSparseFile(prefs, 2); continue; }
if (!strcmp(argument, "--no-sparse")) { LZ4IO_setSparseFile(prefs, 0); continue; }
if (!strcmp(argument, "--favor-decSpeed")) { LZ4IO_favorDecSpeed(prefs, 1); continue; }
@@ -652,7 +654,16 @@ int main(int argc, const char** argv)
/* No output filename ==> try to select one automatically (when possible) */
while ((!output_filename) && (multiple_inputs==0)) {
- if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; break; } /* Default to stdout whenever possible (i.e. not a console) */
+
+ if (!IS_CONSOLE(stdout)) {
+ /* Default to stdout whenever stdout is not the console.
+ * Note : this policy may change in the future, therefore don't rely on it !
+ * To ensure `stdout` is explicitly selected, use `-c` command flag.
+ * Conversely, to ensure output will not become `stdout`, use `-m` command flag */
+ DISPLAYLEVEL(1, "Warning : using stdout as default output. Do not rely on this behavior: use explicit `-c` instead ! \n");
+ output_filename=stdoutmark;
+ break;
+ }
if (mode == om_auto) { /* auto-determine compression or decompression, based on file extension */
mode = determineOpMode(input_filename);
}
@@ -682,10 +693,14 @@ int main(int argc, const char** argv)
break;
}
- /* Check if output is defined as console; trigger an error in this case */
+ if (multiple_inputs==0 && mode != om_list) assert(output_filename);
+ /* when multiple_inputs==1, output_filename may simply be useless,
+ * however, output_filename must be !NULL for next strcmp() tests */
if (!output_filename) output_filename = "*\\dummy^!//";
+
+ /* Check if output is defined as console; trigger an error in this case */
if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) {
- DISPLAYLEVEL(1, "refusing to write to console without -c\n");
+ DISPLAYLEVEL(1, "refusing to write to console without -c \n");
exit(1);
}
/* Downgrade notification level in stdout and multiple file mode */
@@ -701,21 +716,29 @@ int main(int argc, const char** argv)
LZ4IO_setNotificationLevel((int)displayLevel);
if (ifnIdx == 0) multiple_inputs = 0;
if (mode == om_decompress) {
- if (multiple_inputs)
- operationResult = LZ4IO_decompressMultipleFilenames(prefs, inFileNames, ifnIdx, !strcmp(output_filename,stdoutmark) ? stdoutmark : LZ4_EXTENSION);
- else
+ if (multiple_inputs) {
+ assert(ifnIdx <= INT_MAX);
+ operationResult = LZ4IO_decompressMultipleFilenames(prefs, inFileNames, (int)ifnIdx, !strcmp(output_filename,stdoutmark) ? stdoutmark : LZ4_EXTENSION);
+ } else {
operationResult = DEFAULT_DECOMPRESSOR(prefs, input_filename, output_filename);
+ }
+ } else if (mode == om_list){
+ if(!multiple_inputs){
+ inFileNames[ifnIdx++] = input_filename;
+ }
+ operationResult = LZ4IO_displayCompressedFilesInfo(inFileNames, ifnIdx);
+ inFileNames=NULL;
} else { /* compression is default action */
if (legacy_format) {
DISPLAYLEVEL(3, "! Generating LZ4 Legacy format (deprecated) ! \n");
LZ4IO_compressFilename_Legacy(prefs, input_filename, output_filename, cLevel);
} else {
- if (multiple_inputs)
- operationResult = LZ4IO_compressMultipleFilenames(prefs, inFileNames, ifnIdx, !strcmp(output_filename,stdoutmark) ? stdoutmark : LZ4_EXTENSION, cLevel);
- else
+ if (multiple_inputs) {
+ assert(ifnIdx <= INT_MAX);
+ operationResult = LZ4IO_compressMultipleFilenames(prefs, inFileNames, (int)ifnIdx, !strcmp(output_filename,stdoutmark) ? stdoutmark : LZ4_EXTENSION, cLevel);
+ } else {
operationResult = DEFAULT_COMPRESSOR(prefs, input_filename, output_filename, cLevel);
- }
- }
+ } } }
_cleanup:
if (main_pause) waitEnter();
diff --git a/programs/lz4io.c b/programs/lz4io.c
index 121bd44..960c451 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -53,11 +53,11 @@
#include <time.h> /* clock */
#include <sys/types.h> /* stat64 */
#include <sys/stat.h> /* stat64 */
-#include "lz4io.h"
#include "lz4.h" /* still required for legacy format */
#include "lz4hc.h" /* still required for legacy format */
#define LZ4F_STATIC_LINKING_ONLY
#include "lz4frame.h"
+#include "lz4io.h"
/*****************************
@@ -1230,7 +1230,9 @@ int LZ4IO_decompressFilename(LZ4IO_prefs_t* const prefs, const char* input_filen
}
-int LZ4IO_decompressMultipleFilenames(LZ4IO_prefs_t* const prefs, const char** inFileNamesTable, int ifntSize, const char* suffix)
+int LZ4IO_decompressMultipleFilenames(LZ4IO_prefs_t* const prefs,
+ const char** inFileNamesTable, int ifntSize,
+ const char* suffix)
{
int i;
int skippedFiles = 0;
@@ -1250,7 +1252,12 @@ int LZ4IO_decompressMultipleFilenames(LZ4IO_prefs_t* const prefs, const char** i
missingFiles += LZ4IO_decompressSrcFile(prefs, ress, inFileNamesTable[i], stdoutmark);
continue;
}
- if (ofnSize <= ifnSize-suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); if (outFileName==NULL) return ifntSize; }
+ if (ofnSize <= ifnSize-suffixSize+1) {
+ free(outFileName);
+ ofnSize = ifnSize + 20;
+ outFileName = (char*)malloc(ofnSize);
+ if (outFileName==NULL) return ifntSize;
+ }
if (ifnSize <= suffixSize || strcmp(suffixPtr, suffix) != 0) {
DISPLAYLEVEL(1, "File extension doesn't match expected LZ4_EXTENSION (%4s); will not process file: %s\n", suffix, inFileNamesTable[i]);
skippedFiles++;
@@ -1265,3 +1272,137 @@ int LZ4IO_decompressMultipleFilenames(LZ4IO_prefs_t* const prefs, const char** i
free(outFileName);
return missingFiles + skippedFiles;
}
+
+
+/* ********************************************************************* */
+/* ********************** LZ4 --list command *********************** */
+/* ********************************************************************* */
+
+typedef struct {
+ LZ4F_frameInfo_t frameInfo;
+ const char* fileName;
+ unsigned long long fileSize;
+} LZ4IO_cFileInfo_t;
+
+#define LZ4IO_INIT_CFILEINFO { LZ4F_INIT_FRAMEINFO, NULL, 0ULL }
+
+
+typedef enum { LZ4IO_LZ4F_OK, LZ4IO_format_not_known, LZ4IO_not_a_file } LZ4IO_infoResult;
+
+/* This function is limited,
+ * it only works fine for a file consisting of a single valid frame using LZ4 Frame specification.
+ * It will not look at content beyond first frame header.
+ * It's also unable to parse legacy frames, nor skippable ones.
+ *
+ * Things to improve :
+ * - check the entire file for additional content after first frame
+ * + combine results from multiple frames, give total
+ * - Optional :
+ * + report nb of blocks, hence max. possible decompressed size (when not reported in header)
+ */
+static LZ4IO_infoResult
+LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filename)
+{
+ LZ4IO_infoResult result = LZ4IO_format_not_known; /* default result (error) */
+
+ if (!UTIL_isRegFile(input_filename)) return LZ4IO_not_a_file;
+ cfinfo->fileSize = UTIL_getFileSize(input_filename);
+
+ /* Get filename without path prefix */
+ { const char* b = strrchr(input_filename, '/');
+ if (!b) {
+ b = strrchr(input_filename, '\\');
+ }
+ if (b && b != input_filename) {
+ b++;
+ } else {
+ b = input_filename;
+ }
+ cfinfo->fileName = b;
+ }
+
+ /* Read file and extract header */
+ { size_t const hSize = LZ4F_HEADER_SIZE_MAX;
+ size_t readSize=0;
+
+ void* const buffer = malloc(hSize);
+ if (!buffer) EXM_THROW(21, "Allocation error : not enough memory");
+
+ { FILE* const finput = LZ4IO_openSrcFile(input_filename);
+ if (finput) {
+ readSize = fread(buffer, 1, hSize, finput);
+ fclose(finput);
+ } }
+
+ if (readSize > 0) {
+ LZ4F_dctx* dctx;
+ if (!LZ4F_isError(LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION))) {
+ if (!LZ4F_isError(LZ4F_getFrameInfo(dctx, &cfinfo->frameInfo, buffer, &readSize))) {
+ result = LZ4IO_LZ4F_OK;
+ } }
+ LZ4F_freeDecompressionContext(dctx);
+ }
+
+ /* clean */
+ free(buffer);
+ }
+
+ return result;
+}
+
+
+/* buffer : must be a valid memory area of at least 4 bytes */
+const char* LZ4IO_blockTypeID(int sizeID, int blockMode, char* buffer)
+{
+ buffer[0] = 'B';
+ assert(sizeID >= 4); assert(sizeID <=7);
+ buffer[1] = (char)(sizeID + '0');
+ buffer[2] = (blockMode == LZ4F_blockIndependent) ? 'I' : 'D';
+ buffer[3] = 0;
+ return buffer;
+}
+
+
+int LZ4IO_displayCompressedFilesInfo(const char** inFileNames, size_t ifnIdx)
+{
+ int result = 0;
+ size_t idx;
+
+ DISPLAY("%5s %20s %20s %10s %7s %s\n",
+ "Block", "Compressed", "Uncompressed", "Ratio", "Check", "Filename");
+
+ for (idx=0; idx<ifnIdx; idx++) {
+ /* Get file info */
+ LZ4IO_cFileInfo_t cfinfo = LZ4IO_INIT_CFILEINFO;
+ LZ4IO_infoResult const op_result = LZ4IO_getCompressedFileInfo(&cfinfo, inFileNames[idx]);
+ if (op_result != LZ4IO_LZ4F_OK) {
+ if (op_result == LZ4IO_not_a_file) {
+ DISPLAYLEVEL(1, "lz4: %s is not a regular file \n", inFileNames[idx]);
+ } else {
+ assert(op_result == LZ4IO_format_not_known);
+ DISPLAYLEVEL(1, "lz4: %s: File format not recognized \n", inFileNames[idx]);
+ }
+ result = 1;
+ continue;
+ }
+ if (cfinfo.frameInfo.contentSize) {
+ char buffer[5];
+ double const ratio = (double)cfinfo.fileSize / cfinfo.frameInfo.contentSize;
+ DISPLAY("%5s %20llu %20llu %8.4f %7s %s \n",
+ LZ4IO_blockTypeID(cfinfo.frameInfo.blockSizeID, cfinfo.frameInfo.blockMode, buffer),
+ cfinfo.fileSize,
+ cfinfo.frameInfo.contentSize, ratio,
+ cfinfo.frameInfo.contentChecksumFlag ? "XXH32" : "-",
+ cfinfo.fileName);
+ } else {
+ char buffer[5];
+ DISPLAY("%5s %20llu %20s %10s %7s %s \n",
+ LZ4IO_blockTypeID(cfinfo.frameInfo.blockSizeID, cfinfo.frameInfo.blockMode, buffer),
+ cfinfo.fileSize,
+ "-", "-",
+ cfinfo.frameInfo.contentChecksumFlag ? "XXH32" : "-",
+ cfinfo.fileName);
+ }
+ }
+ return result;
+}
diff --git a/programs/lz4io.h b/programs/lz4io.h
index 54d49be..213dc95 100644
--- a/programs/lz4io.h
+++ b/programs/lz4io.h
@@ -124,4 +124,9 @@ void LZ4IO_setRemoveSrcFile(LZ4IO_prefs_t* const prefs, unsigned flag);
void LZ4IO_favorDecSpeed(LZ4IO_prefs_t* const prefs, int favor);
+/* implement --list
+ * @return 0 on success, 1 on error */
+int LZ4IO_displayCompressedFilesInfo(const char** inFileNames, size_t ifnIdx);
+
+
#endif /* LZ4IO_H_237902873 */
diff --git a/programs/util.h b/programs/util.h
index 6a35481..1dd515c 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -379,7 +379,7 @@ UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFi
*/
UTIL_STATIC void* UTIL_realloc(void* ptr, size_t size)
{
- void* newptr = realloc(ptr, size);
+ void* const newptr = realloc(ptr, size);
if (newptr) return newptr;
free(ptr);
return NULL;
@@ -529,7 +529,8 @@ UTIL_STATIC int UTIL_prepareFileList(const char* dirName, char** bufStart, size_
* In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called.
*/
UTIL_STATIC const char**
-UTIL_createFileList(const char** inputNames, unsigned inputNamesNb, char** allocatedBuffer, unsigned* allocatedNamesNb)
+UTIL_createFileList(const char** inputNames, unsigned inputNamesNb,
+ char** allocatedBuffer, unsigned* allocatedNamesNb)
{
size_t pos;
unsigned i, nbFiles;
@@ -543,16 +544,14 @@ UTIL_createFileList(const char** inputNames, unsigned inputNamesNb, char** alloc
if (!UTIL_isDirectory(inputNames[i])) {
size_t const len = strlen(inputNames[i]);
if (pos + len >= bufSize) {
- size_t newListSize = bufSize + LIST_SIZE_INCREASE;
- buf = (char*)UTIL_realloc(buf, newListSize);
- bufSize = newListSize;
+ while (pos + len >= bufSize) bufSize += LIST_SIZE_INCREASE;
+ buf = (char*)UTIL_realloc(buf, bufSize);
if (!buf) return NULL;
}
- if (pos + len < bufSize) {
- strncpy(buf + pos, inputNames[i], bufSize - pos);
- pos += len + 1;
- nbFiles++;
- }
+ assert(pos + len < bufSize);
+ strncpy(buf + pos, inputNames[i], bufSize - pos);
+ pos += len + 1;
+ nbFiles++;
} else {
char* bufend = buf + bufSize;
nbFiles += (unsigned)UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend);
diff --git a/tests/Makefile b/tests/Makefile
index 70cae63..67514e4 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -43,15 +43,8 @@ CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
CPPFLAGS+= -I$(LZ4DIR) -I$(PRGDIR) -DXXH_NAMESPACE=LZ4_
FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
+include ../Makefile.inc
-# Define *.exe as extension for Windows systems
-ifneq (,$(filter Windows%,$(OS)))
-EXT =.exe
-VOID = nul
-else
-EXT =
-VOID = /dev/null
-endif
LZ4 := $(PRGDIR)/lz4$(EXT)
@@ -60,7 +53,6 @@ TEST_FILES := COPYING
FUZZER_TIME := -T90s
NB_LOOPS ?= -i1
-
default: all
all: fullbench fuzzer frametest roundTripTest datagen checkFrame
@@ -75,7 +67,7 @@ lib liblz4.pc:
$(MAKE) -C $(LZ4DIR) $@ CFLAGS="$(CFLAGS)"
lz4c unlz4 lz4cat: lz4
- ln -sf $(LZ4) $(PRGDIR)/$@
+ $(LN_SF) $(LZ4) $(PRGDIR)/$@
lz4c32: # create a 32-bits version for 32/64 interop tests
$(MAKE) -C $(PRGDIR) $@ CFLAGS="-m32 $(CFLAGS)"
@@ -95,7 +87,7 @@ fullbench-lib: fullbench.c $(LZ4DIR)/liblz4.a
fullbench-dll: fullbench.c $(LZ4DIR)/xxhash.c
$(MAKE) -C $(LZ4DIR) liblz4
- $(CC) $(FLAGS) $^ -o $@$(EXT) -DLZ4_DLL_IMPORT=1 $(LZ4DIR)/dll/liblz4.dll
+ $(CC) $(FLAGS) $^ -o $@$(EXT) -DLZ4_DLL_IMPORT=1 $(LZ4DIR)/dll/$(LIBLZ4).dll
fuzzer : lz4.o lz4hc.o xxhash.o fuzzer.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
@@ -137,13 +129,15 @@ checkTag: checkTag.c $(LZ4DIR)/lz4.h
#-----------------------------------------------------------------------------
# validated only for Linux, OSX, BSD, Hurd and Solaris targets
#-----------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku MidnightBSD))
+ifeq ($(POSIX_ENV),Yes)
MD5:=md5sum
ifneq (,$(filter $(shell uname), Darwin ))
MD5:=md5 -r
endif
+# note : we should probably settle on a single compare utility
+CMP:=cmp
DIFF:=diff
ifneq (,$(filter $(shell uname),SunOS))
DIFF:=gdiff
@@ -176,15 +170,15 @@ test-install: lz4 lib liblz4.pc
test-lz4-sparse: lz4 datagen
@echo "\n ---- test sparse file support ----"
./datagen -g5M -P100 > tmplsdg5M
- $(LZ4) -B4D tmplsdg5M | $(LZ4) -dv --sparse > tmplscB4
+ $(LZ4) -B4D tmplsdg5M -c | $(LZ4) -dv --sparse > tmplscB4
$(DIFF) -s tmplsdg5M tmplscB4
- $(LZ4) -B5D tmplsdg5M | $(LZ4) -dv --sparse > tmplscB5
+ $(LZ4) -B5D tmplsdg5M -c | $(LZ4) -dv --sparse > tmplscB5
$(DIFF) -s tmplsdg5M tmplscB5
- $(LZ4) -B6D tmplsdg5M | $(LZ4) -dv --sparse > tmplscB6
+ $(LZ4) -B6D tmplsdg5M -c | $(LZ4) -dv --sparse > tmplscB6
$(DIFF) -s tmplsdg5M tmplscB6
- $(LZ4) -B7D tmplsdg5M | $(LZ4) -dv --sparse > tmplscB7
+ $(LZ4) -B7D tmplsdg5M -c | $(LZ4) -dv --sparse > tmplscB7
$(DIFF) -s tmplsdg5M tmplscB7
- $(LZ4) tmplsdg5M | $(LZ4) -dv --no-sparse > tmplsnosparse
+ $(LZ4) tmplsdg5M -c | $(LZ4) -dv --no-sparse > tmplsnosparse
$(DIFF) -s tmplsdg5M tmplsnosparse
ls -ls tmpls*
./datagen -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > tmplsodd # Odd size file (to generate non-full last block)
@@ -200,7 +194,7 @@ test-lz4-sparse: lz4 datagen
cat tmplsdg1M tmplsdg1M > tmpls2M
$(LZ4) -B5 -v tmplsdg1M tmplsc
$(LZ4) -d -v tmplsc tmplsr
- $(LZ4) -d -v tmplsc >> tmplsr
+ $(LZ4) -d -v tmplsc -c >> tmplsr
ls -ls tmp*
$(DIFF) tmpls2M tmplsr
@$(RM) tmpls*
@@ -208,8 +202,8 @@ test-lz4-sparse: lz4 datagen
test-lz4-contentSize: lz4 datagen
@echo "\n ---- test original size support ----"
./datagen -g15M > tmplc1
- $(LZ4) -v tmplc1 | $(LZ4) -t
- $(LZ4) -v --content-size tmplc1 | $(LZ4) -d > tmplc2
+ $(LZ4) -v tmplc1 -c | $(LZ4) -t
+ $(LZ4) -v --content-size tmplc1 -c | $(LZ4) -d > tmplc2
$(DIFF) -s tmplc1 tmplc2
@$(RM) tmplc*
@@ -218,11 +212,11 @@ test-lz4-frame-concatenation: lz4 datagen
@echo -n > tmp-lfc-empty
@echo hi > tmp-lfc-nonempty
cat tmp-lfc-nonempty tmp-lfc-empty tmp-lfc-nonempty > tmp-lfc-src
- @$(LZ4) -zq tmp-lfc-empty > tmp-lfc-empty.lz4
- @$(LZ4) -zq tmp-lfc-nonempty > tmp-lfc-nonempty.lz4
+ $(LZ4) -zq tmp-lfc-empty -c > tmp-lfc-empty.lz4
+ $(LZ4) -zq tmp-lfc-nonempty -c > tmp-lfc-nonempty.lz4
cat tmp-lfc-nonempty.lz4 tmp-lfc-empty.lz4 tmp-lfc-nonempty.lz4 > tmp-lfc-concat.lz4
- $(LZ4) -d tmp-lfc-concat.lz4 > tmp-lfc-result
- cmp tmp-lfc-src tmp-lfc-result
+ $(LZ4) -d tmp-lfc-concat.lz4 -c > tmp-lfc-result
+ $(CMP) tmp-lfc-src tmp-lfc-result
@$(RM) tmp-lfc-*
@echo frame concatenation test completed
@@ -241,15 +235,15 @@ test-lz4-multiple: lz4 datagen
mv tmp-tlm2 tmp-tlm2-orig
mv tmp-tlm3 tmp-tlm3-orig
$(LZ4) -d -f -m tmp-tlm*.lz4
- cmp tmp-tlm1 tmp-tlm1-orig # must be identical
- cmp tmp-tlm2 tmp-tlm2-orig
- cmp tmp-tlm3 tmp-tlm3-orig
+ $(CMP) tmp-tlm1 tmp-tlm1-orig # must be identical
+ $(CMP) tmp-tlm2 tmp-tlm2-orig
+ $(CMP) tmp-tlm3 tmp-tlm3-orig
# compress multiple files into stdout
cat tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 > tmp-tlm-concat1
$(RM) *.lz4
$(LZ4) -m tmp-tlm1 tmp-tlm2 tmp-tlm3 -c > tmp-tlm-concat2
test ! -f tmp-tlm1.lz4 # must not create .lz4 artefact
- cmp tmp-tlm-concat1 tmp-tlm-concat2 # must be equivalent
+ $(CMP) tmp-tlm-concat1 tmp-tlm-concat2 # must be equivalent
# decompress multiple files into stdout
$(RM) tmp-tlm-concat1 tmp-tlm-concat2
$(LZ4) -f -m tmp-tlm1 tmp-tlm2 tmp-tlm3 # generate .lz4 to decompress
@@ -257,7 +251,7 @@ test-lz4-multiple: lz4 datagen
$(RM) tmp-tlm1 tmp-tlm2 tmp-tlm3
$(LZ4) -d -m tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 -c > tmp-tlm-concat2
test ! -f tmp-tlm1 # must not create file artefact
- cmp tmp-tlm-concat1 tmp-tlm-concat2 # must be equivalent
+ $(CMP) tmp-tlm-concat1 tmp-tlm-concat2 # must be equivalent
# compress multiple files, one of which is absent (must fail)
! $(LZ4) -f -m tmp-tlm-concat1 notHere tmp-tlm-concat2 # must fail : notHere not present
@$(RM) tmp-tlm*
@@ -303,6 +297,7 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat
test ! -f ./-z
$(DIFF) -q tmp-tlb-hw tmp-tlb4
$(LZ4) -f tmp-tlb-hw
+ $(LZ4) --list tmp-tlb-hw.lz4 # test --list on valid single-frame file
cat tmp-tlb-hw >> tmp-tlb-hw.lz4
$(LZ4) -f tmp-tlb-hw.lz4 # uncompress valid frame followed by invalid data
$(LZ4) -BX tmp-tlb-hw -c -q | $(LZ4) -tv # test block checksum
@@ -315,7 +310,7 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat
! $(LZ4) -c --fast=-1 tmp-tlb-dg20K # lz4 should fail when fast=-1
# Test for #596
@echo "TEST" > tmp-tlb-test
- $(LZ4) tmp-tlb-test
+ $(LZ4) -m tmp-tlb-test
$(LZ4) tmp-tlb-test.lz4 tmp-tlb-test2
$(DIFF) -q tmp-tlb-test tmp-tlb-test2
@$(RM) tmp-tlb*
@@ -341,8 +336,8 @@ test-lz4-dict: lz4 datagen
for l in 0 1 4 128 32767 32768 32769 65535 65536 65537 98303 98304 98305 131071 131072 131073; do \
./datagen -g$$l > tmp-dict-$$l; \
$(DD) if=tmp-dict-$$l of=tmp-dict-$$l-tail bs=1 count=65536 skip=$$((l > 65536 ? l - 65536 : 0)); \
- < tmp-dict-$$l $(LZ4) -D stdin tmp-dict-data-128KB | $(LZ4) -dD tmp-dict-$$l-tail | $(DIFF) - tmp-dict-data-128KB; \
- < tmp-dict-$$l-tail $(LZ4) -D stdin tmp-dict-data-128KB | $(LZ4) -dD tmp-dict-$$l | $(DIFF) - tmp-dict-data-128KB; \
+ < tmp-dict-$$l $(LZ4) -D stdin tmp-dict-data-128KB -c | $(LZ4) -dD tmp-dict-$$l-tail | $(DIFF) - tmp-dict-data-128KB; \
+ < tmp-dict-$$l-tail $(LZ4) -D stdin tmp-dict-data-128KB -c | $(LZ4) -dD tmp-dict-$$l | $(DIFF) - tmp-dict-data-128KB; \
done
@$(RM) tmp-dict*
diff --git a/tests/checkFrame.c b/tests/checkFrame.c
index 50c0405..139a599 100644
--- a/tests/checkFrame.c
+++ b/tests/checkFrame.c
@@ -105,7 +105,7 @@ typedef struct {
LZ4F_decompressionContext_t ctx;
} cRess_t;
-static int createCResources(cRess_t *ress)
+static int createCResources(cRess_t* ress)
{
ress->srcBufferSize = 4 MB;
ress->srcBuffer = malloc(ress->srcBufferSize);
diff --git a/tests/frametest.c b/tests/frametest.c
index a5197ff..bf95beb 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -167,7 +167,7 @@ static unsigned FUZ_highbit(U32 v32)
/*-*******************************************************
* Tests
*********************************************************/
-#define CHECK_V(v,f) v = f; if (LZ4F_isError(v)) { fprintf(stderr, "%s\n", LZ4F_getErrorName(v)); goto _output_error; }
+#define CHECK_V(v,f) v = f; if (LZ4F_isError(v)) { fprintf(stderr, "%s \n", LZ4F_getErrorName(v)); goto _output_error; }
#define CHECK(f) { LZ4F_errorCode_t const CHECK_V(err_ , f); }
int basicTests(U32 seed, double compressibility)
@@ -667,8 +667,8 @@ int basicTests(U32 seed, double compressibility)
for (blockSizeID = 4; blockSizeID < 8; ++blockSizeID) {
result = LZ4F_getBlockSize(blockSizeID);
CHECK(result);
- DISPLAYLEVEL(3, "Returned block size of %zu bytes for blockID %u \n",
- result, blockSizeID);
+ DISPLAYLEVEL(3, "Returned block size of %u bytes for blockID %u \n",
+ (unsigned)result, blockSizeID);
}
/* Test an invalid input that's too large */
@@ -770,7 +770,8 @@ static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, un
const BYTE* b2=(const BYTE*)buff2;
DISPLAY("locateBuffDiff: looking for error position \n");
if (nonContiguous) {
- DISPLAY("mode %u: non-contiguous output (%zu bytes), cannot search \n", nonContiguous, size);
+ DISPLAY("mode %u: non-contiguous output (%u bytes), cannot search \n",
+ nonContiguous, (unsigned)size);
return;
}
while (p < size && b1[p]==b2[p]) p++;
@@ -795,8 +796,9 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
clock_t const startClock = clock();
clock_t const clockDuration = duration_s * CLOCKS_PER_SEC;
# undef CHECK
-# define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
- DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
+# define EXIT_MSG(...) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
+ DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
+# define CHECK(cond, ...) { if (cond) { EXIT_MSG(__VA_ARGS__); } }
/* Create buffers */
{ size_t const creationStatus = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
@@ -950,9 +952,10 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
CHECK(decSize != 0, "Frame decompression failed (error %i)", (int)decSize);
if (totalOut) { /* otherwise, it's a skippable frame */
U64 const crcDecoded = XXH64_digest(&xxh64);
- if (crcDecoded != crcOrig) locateBuffDiff(srcStart, decodedBuffer, srcSize, nonContiguousDst);
- CHECK(crcDecoded != crcOrig, "Decompression corruption");
- }
+ if (crcDecoded != crcOrig) {
+ locateBuffDiff(srcStart, decodedBuffer, srcSize, nonContiguousDst);
+ EXIT_MSG("Decompression corruption");
+ } }
}
}
diff --git a/tests/fullbench.c b/tests/fullbench.c
index 1a52aab..d2af662 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -162,13 +162,13 @@ static LZ4_stream_t LZ4_stream;
static void local_LZ4_resetDictT(void)
{
void* const r = LZ4_initStream(&LZ4_stream, sizeof(LZ4_stream));
- assert(r != NULL);
+ assert(r != NULL); (void)r;
}
static void local_LZ4_createStream(void)
{
void* const r = LZ4_initStream(&LZ4_stream, sizeof(LZ4_stream));
- assert(r != NULL);
+ assert(r != NULL); (void)r;
}
static int local_LZ4_saveDict(const char* in, char* out, int inSize)
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 3128e6d..a5b5c93 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -50,6 +50,10 @@
#include <assert.h>
#include <limits.h> /* INT_MAX */
+#if defined(_AIX)
+# include <sys/mman.h> /* mmap */
+#endif
+
#define LZ4_DISABLE_DEPRECATE_WARNINGS /* LZ4_decompress_fast */
#define LZ4_STATIC_LINKING_ONLY
#include "lz4.h"
@@ -315,20 +319,21 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
void* const lowAddrBuffer = FUZ_createLowAddr(labSize);
void* const stateLZ4 = malloc((size_t)LZ4_sizeofState());
void* const stateLZ4HC = malloc((size_t)LZ4_sizeofStateHC());
- LZ4_stream_t LZ4dict;
- LZ4_streamHC_t LZ4dictHC;
+ LZ4_stream_t LZ4dictBody;
+ LZ4_streamHC_t* LZ4dictHC = LZ4_createStreamHC();
U32 coreRandState = seed;
clock_t const clockStart = clock();
clock_t const clockDuration = (clock_t)duration_s * CLOCKS_PER_SEC;
int result = 0;
unsigned cycleNb;
-# define FUZ_CHECKTEST(cond, ...) \
- if (cond) { \
- printf("Test %u : ", testNb); printf(__VA_ARGS__); \
- printf(" (seed %u, cycle %u) \n", seed, cycleNb); \
- goto _output_error; \
- }
+# define EXIT_MSG(...) { \
+ printf("Test %u : ", testNb); printf(__VA_ARGS__); \
+ printf(" (seed %u, cycle %u) \n", seed, cycleNb); \
+ exit(1); \
+}
+
+# define FUZ_CHECKTEST(cond, ...) { if (cond) { EXIT_MSG(__VA_ARGS__) } }
# define FUZ_DISPLAYTEST(...) { \
testNb++; \
@@ -341,12 +346,11 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
/* init */
- if(!CNBuffer || !compressedBuffer || !decodedBuffer) {
+ if(!CNBuffer || !compressedBuffer || !decodedBuffer || !LZ4dictHC) {
DISPLAY("Not enough memory to start fuzzer tests");
- goto _output_error;
+ exit(1);
}
- if ( LZ4_initStream(&LZ4dict, sizeof(LZ4dict)) == NULL) abort();
- if ( LZ4_initStreamHC(&LZ4dictHC, sizeof(LZ4dictHC)) == NULL) abort();
+ if ( LZ4_initStream(&LZ4dictBody, sizeof(LZ4dictBody)) == NULL) abort();
{ U32 randState = coreRandState ^ PRIME3;
FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
}
@@ -450,133 +454,126 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
/* Test compression HC */
FUZ_DISPLAYTEST("test LZ4_compress_HC()");
- ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
- FUZ_CHECKTEST(ret==0, "LZ4_compress_HC() failed");
- HCcompressedSize = ret;
+ HCcompressedSize = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
+ FUZ_CHECKTEST(HCcompressedSize==0, "LZ4_compress_HC() failed");
/* Test compression HC using external state */
FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC()");
- ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
- FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed")
+ { int const r = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
+ FUZ_CHECKTEST(r==0, "LZ4_compress_HC_extStateHC() failed")
+ }
/* Test compression HC using fast reset external state */
FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC_fastReset()");
- ret = LZ4_compress_HC_extStateHC_fastReset(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
- FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC_fastReset() failed");
+ { int const r = LZ4_compress_HC_extStateHC_fastReset(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
+ FUZ_CHECKTEST(r==0, "LZ4_compress_HC_extStateHC_fastReset() failed");
+ }
/* Test compression using external state */
FUZ_DISPLAYTEST("test LZ4_compress_fast_extState()");
- ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
- FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed");
+ { int const r = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
+ FUZ_CHECKTEST(r==0, "LZ4_compress_fast_extState() failed"); }
/* Test compression using fast reset external state*/
FUZ_DISPLAYTEST();
- ret = LZ4_compress_fast_extState_fastReset(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
- FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState_fastReset() failed");
+ { int const r = LZ4_compress_fast_extState_fastReset(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
+ FUZ_CHECKTEST(r==0, "LZ4_compress_fast_extState_fastReset() failed"); }
/* Test compression */
FUZ_DISPLAYTEST("test LZ4_compress_default()");
- ret = LZ4_compress_default(block, compressedBuffer, blockSize, (int)compressedBufferSize);
- FUZ_CHECKTEST(ret==0, "LZ4_compress_default() failed");
- compressedSize = ret;
+ compressedSize = LZ4_compress_default(block, compressedBuffer, blockSize, (int)compressedBufferSize);
+ FUZ_CHECKTEST(compressedSize<=0, "LZ4_compress_default() failed");
/* Decompression tests */
- /* Test decoding with output size exactly correct => must work */
- FUZ_DISPLAYTEST("LZ4_decompress_fast() with exact output buffer");
- ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize);
- FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space");
- FUZ_CHECKTEST(ret!=compressedSize, "LZ4_decompress_fast failed : did not fully read compressed data");
- { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
- FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data");
- }
+ /* Test decompress_fast() with input buffer size exactly correct => must not read out of bound */
+ { char* const cBuffer_exact = (char*)malloc((size_t)compressedSize);
+ assert(cBuffer_exact != NULL);
+ assert(compressedSize <= (int)compressedBufferSize);
+ memcpy(cBuffer_exact, compressedBuffer, compressedSize);
+
+ /* Test decoding with output size exactly correct => must work */
+ FUZ_DISPLAYTEST("LZ4_decompress_fast() with exact output buffer");
+ { int const r = LZ4_decompress_fast(cBuffer_exact, decodedBuffer, blockSize);
+ FUZ_CHECKTEST(r<0, "LZ4_decompress_fast failed despite correct space");
+ FUZ_CHECKTEST(r!=compressedSize, "LZ4_decompress_fast failed : did not fully read compressed data");
+ }
+ { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0);
+ FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data");
+ }
- /* Test decoding with one byte missing => must fail */
- FUZ_DISPLAYTEST("LZ4_decompress_fast() with output buffer 1-byte too short");
- decodedBuffer[blockSize-1] = 0;
- ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1);
- FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
- FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer");
+ /* Test decoding with one byte missing => must fail */
+ FUZ_DISPLAYTEST("LZ4_decompress_fast() with output buffer 1-byte too short");
+ decodedBuffer[blockSize-1] = 0;
+ { int const r = LZ4_decompress_fast(cBuffer_exact, decodedBuffer, blockSize-1);
+ FUZ_CHECKTEST(r>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
+ }
+ FUZ_CHECKTEST(decodedBuffer[blockSize-1]!=0, "LZ4_decompress_fast overrun specified output buffer");
- /* Test decoding with one byte too much => must fail */
- FUZ_DISPLAYTEST();
- ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1);
- FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");
+ /* Test decoding with one byte too much => must fail */
+ FUZ_DISPLAYTEST();
+ { int const r = LZ4_decompress_fast(cBuffer_exact, decodedBuffer, blockSize+1);
+ FUZ_CHECKTEST(r>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");
+ }
- /* Test decoding with empty input */
- FUZ_DISPLAYTEST("LZ4_decompress_safe() with empty input");
- LZ4_decompress_safe(compressedBuffer, decodedBuffer, 0, blockSize);
+ /* Test decoding with output size exactly what's necessary => must work */
+ FUZ_DISPLAYTEST();
+ decodedBuffer[blockSize] = 0;
+ { int const r = LZ4_decompress_safe(cBuffer_exact, decodedBuffer, compressedSize, blockSize);
+ FUZ_CHECKTEST(r<0, "LZ4_decompress_safe failed despite sufficient space");
+ FUZ_CHECKTEST(r!=blockSize, "LZ4_decompress_safe did not regenerate original data");
+ }
+ FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
+ { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+ FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
+ }
- /* Test decoding with a one byte input */
- FUZ_DISPLAYTEST("LZ4_decompress_safe() with one byte input");
- { char const tmp = (char)0xFF;
- LZ4_decompress_safe(&tmp, decodedBuffer, 1, blockSize);
- }
+ /* Test decoding with more than enough output size => must work */
+ FUZ_DISPLAYTEST();
+ decodedBuffer[blockSize] = 0;
+ decodedBuffer[blockSize+1] = 0;
+ { int const r = LZ4_decompress_safe(cBuffer_exact, decodedBuffer, compressedSize, blockSize+1);
+ FUZ_CHECKTEST(r<0, "LZ4_decompress_safe failed despite amply sufficient space");
+ FUZ_CHECKTEST(r!=blockSize, "LZ4_decompress_safe did not regenerate original data");
+ }
+ FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ4_decompress_safe overrun specified output buffer size");
+ { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0);
+ FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
+ }
- /* Test decoding shortcut edge case */
- FUZ_DISPLAYTEST("LZ4_decompress_safe() with shortcut edge case");
- { char tmp[17];
- /* 14 bytes of literals, followed by a 14 byte match.
- * Should not read beyond the end of the buffer.
- * See https://github.com/lz4/lz4/issues/508. */
- *tmp = (char)0xEE;
- memset(tmp + 1, 0, 14);
- tmp[15] = 14;
- tmp[16] = 0;
- ret = LZ4_decompress_safe(tmp, decodedBuffer, sizeof(tmp), blockSize);
- FUZ_CHECKTEST(ret >= 0, "LZ4_decompress_safe() should fail");
- }
+ /* Test decoding with output size being one byte too short => must fail */
+ FUZ_DISPLAYTEST();
+ decodedBuffer[blockSize-1] = 0;
+ { int const r = LZ4_decompress_safe(cBuffer_exact, decodedBuffer, compressedSize, blockSize-1);
+ FUZ_CHECKTEST(r>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short");
+ }
+ FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe overrun specified output buffer size");
+ /* Test decoding with output size being 10 bytes too short => must fail */
+ FUZ_DISPLAYTEST();
+ if (blockSize>10) {
+ decodedBuffer[blockSize-10] = 0;
+ { int const r = LZ4_decompress_safe(cBuffer_exact, decodedBuffer, compressedSize, blockSize-10);
+ FUZ_CHECKTEST(r>=0, "LZ4_decompress_safe should have failed, due to Output Size being 10 bytes too short");
+ }
+ FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe overrun specified output buffer size");
+ }
- /* Test decoding with output size exactly what's necessary => must work */
- FUZ_DISPLAYTEST();
- decodedBuffer[blockSize] = 0;
- ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize);
- FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space");
- FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data");
- FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
- { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
- FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
+ free(cBuffer_exact);
}
- // Test decoding with more than enough output size => must work
+ /* Test decoding with input size being one byte too short => must fail */
FUZ_DISPLAYTEST();
- decodedBuffer[blockSize] = 0;
- decodedBuffer[blockSize+1] = 0;
- ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize+1);
- FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite amply sufficient space");
- FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data");
- FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ4_decompress_safe overrun specified output buffer size");
- { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0);
- FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
+ { int const r = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize);
+ FUZ_CHECKTEST(r>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize);
}
- // Test decoding with output size being one byte too short => must fail
- FUZ_DISPLAYTEST();
- decodedBuffer[blockSize-1] = 0;
- ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-1);
- FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short");
- FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe overrun specified output buffer size");
-
- // Test decoding with output size being 10 bytes too short => must fail
- FUZ_DISPLAYTEST();
- if (blockSize>10) {
- decodedBuffer[blockSize-10] = 0;
- ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-10);
- FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being 10 bytes too short");
- FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe overrun specified output buffer size");
- }
-
- // Test decoding with input size being one byte too short => must fail
- FUZ_DISPLAYTEST();
- ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize);
- FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize);
-
- // Test decoding with input size being one byte too large => must fail
+ /* Test decoding with input size being one byte too large => must fail */
FUZ_DISPLAYTEST();
decodedBuffer[blockSize] = 0;
- ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize+1, blockSize);
- FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large");
+ { int const r = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize+1, blockSize);
+ FUZ_CHECKTEST(r>=0, "LZ4_decompress_safe should have failed, due to input size being too large");
+ }
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
/* Test partial decoding => must work */
@@ -654,8 +651,10 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer+dictSize, blockSize, decodedBuffer, dictSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
{ U32 const crcCheck = XXH32(decodedBuffer+dictSize, (size_t)blockSize, 0);
- if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
- FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
+ if (crcCheck!=crcOrig) {
+ FUZ_findDiff(block, decodedBuffer);
+ EXIT_MSG("LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
+ }
}
FUZ_DISPLAYTEST("test LZ4_decompress_safe_usingDict()");
@@ -669,20 +668,20 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_DISPLAYTEST("test LZ4_compress_fast_continue(), with non-contiguous dictionary");
dict -= (size_t)(FUZ_rand(&randState) & 0xF) + 1; /* create space, so now dictionary is an ExtDict */
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
- LZ4_loadDict(&LZ4dict, dict, dictSize);
- blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
+ LZ4_loadDict(&LZ4dictBody, dict, dictSize);
+ blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue failed");
FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary but with an output buffer too short by one byte");
- LZ4_loadDict(&LZ4dict, dict, dictSize);
- ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
+ LZ4_loadDict(&LZ4dictBody, dict, dictSize);
+ ret = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary loaded with LZ4_loadDict()");
DISPLAYLEVEL(5, " compress %i bytes from buffer(%p) into dst(%p) using dict(%p) of size %i \n",
blockSize, (const void *)block, (void *)decodedBuffer, (const void *)dict, dictSize);
- LZ4_loadDict(&LZ4dict, dict, dictSize);
- ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
+ LZ4_loadDict(&LZ4dictBody, dict, dictSize);
+ ret = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue should work : enough size available within output buffer");
@@ -694,9 +693,11 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
- { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
- if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
- FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
+ { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0);
+ if (crcCheck!=crcOrig) {
+ FUZ_findDiff(block, decodedBuffer);
+ EXIT_MSG("LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
+ }
}
FUZ_DISPLAYTEST();
@@ -736,15 +737,15 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
U32 expectedCrc;
FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_loadDict()");
- LZ4_loadDict(&LZ4dict, dict, dictSize);
- expectedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
+ LZ4_loadDict(&LZ4dictBody, dict, dictSize);
+ expectedSize = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
FUZ_CHECKTEST(expectedSize<=0, "LZ4_compress_fast_continue reference compression for extDictCtx should have succeeded");
expectedCrc = XXH32(compressedBuffer, expectedSize, 0);
FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary()");
- LZ4_loadDict(&LZ4dict, dict, dictSize);
+ LZ4_loadDict(&LZ4dictBody, dict, dictSize);
LZ4_initStream(&LZ4_stream, sizeof(LZ4_stream));
- LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
+ LZ4_attach_dictionary(&LZ4_stream, &LZ4dictBody);
blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue using extDictCtx failed");
FUZ_CHECKTEST(LZ4_stream.internal_donotuse.dirty, "context should be good");
@@ -759,14 +760,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary(), but output buffer is 1 byte too short");
LZ4_resetStream_fast(&LZ4_stream);
- LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
+ LZ4_attach_dictionary(&LZ4_stream, &LZ4dictBody);
ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using extDictCtx should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
/* note : context is no longer dirty after a failed compressed block */
FUZ_DISPLAYTEST();
LZ4_resetStream_fast(&LZ4_stream);
- LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
+ LZ4_attach_dictionary(&LZ4_stream, &LZ4dictBody);
ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx should work : enough size available within output buffer");
@@ -776,7 +777,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_DISPLAYTEST();
LZ4_resetStream_fast(&LZ4_stream);
- LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
+ LZ4_attach_dictionary(&LZ4_stream, &LZ4dictBody);
ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx with re-used context should work : enough size available within output buffer");
@@ -792,8 +793,10 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
{ U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
- if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
- FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
+ if (crcCheck!=crcOrig) {
+ FUZ_findDiff(block, decodedBuffer);
+ EXIT_MSG("LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
+ }
}
FUZ_DISPLAYTEST();
@@ -830,24 +833,24 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_DISPLAYTEST("LZ4_compress_HC_continue with an external dictionary");
dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
- LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
- LZ4_setCompressionLevel (&LZ4dictHC, compressionLevel);
- blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize);
+ LZ4_loadDictHC(LZ4dictHC, dict, dictSize);
+ LZ4_setCompressionLevel (LZ4dictHC, compressionLevel);
+ blockContinueCompressedSize = LZ4_compress_HC_continue(LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue failed");
- FUZ_CHECKTEST(LZ4dictHC.internal_donotuse.dirty, "Context should be clean");
+ FUZ_CHECKTEST(LZ4dictHC->internal_donotuse.dirty, "Context should be clean");
FUZ_DISPLAYTEST("LZ4_compress_HC_continue with same external dictionary, but output buffer 1 byte too short");
- LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
- ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
+ LZ4_loadDictHC(LZ4dictHC, dict, dictSize);
+ ret = LZ4_compress_HC_continue(LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDict should fail : one missing byte for output buffer (expected %i, but result=%i)", blockContinueCompressedSize, ret);
/* note : context is no longer dirty after a failed compressed block */
FUZ_DISPLAYTEST("LZ4_compress_HC_continue with same external dictionary, and output buffer exactly the right size");
- LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
- ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
+ LZ4_loadDictHC(LZ4dictHC, dict, dictSize);
+ ret = LZ4_compress_HC_continue(LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue size is different : ret(%i) != expected(%i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue should work : enough size available within output buffer");
- FUZ_CHECKTEST(LZ4dictHC.internal_donotuse.dirty, "Context should be clean");
+ FUZ_CHECKTEST(LZ4dictHC->internal_donotuse.dirty, "Context should be clean");
FUZ_DISPLAYTEST();
decodedBuffer[blockSize] = 0;
@@ -855,45 +858,46 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
{ U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0);
- if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
- FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
- }
+ if (crcCheck!=crcOrig) {
+ FUZ_findDiff(block, decodedBuffer);
+ EXIT_MSG("LZ4_decompress_safe_usingDict corrupted decoded data");
+ } }
/* Compress HC using external dictionary stream */
FUZ_DISPLAYTEST();
- {
- LZ4_streamHC_t LZ4_streamHC;
- LZ4_initStreamHC(&LZ4_streamHC, sizeof(LZ4_streamHC));
+ { LZ4_streamHC_t* const LZ4_streamHC = LZ4_createStreamHC();
- LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
- LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC);
- LZ4_setCompressionLevel (&LZ4_streamHC, compressionLevel);
- blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, (int)compressedBufferSize);
+ LZ4_loadDictHC(LZ4dictHC, dict, dictSize);
+ LZ4_attach_HC_dictionary(LZ4_streamHC, LZ4dictHC);
+ LZ4_setCompressionLevel (LZ4_streamHC, compressionLevel);
+ blockContinueCompressedSize = LZ4_compress_HC_continue(LZ4_streamHC, block, compressedBuffer, blockSize, (int)compressedBufferSize);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue with ExtDictCtx failed");
- FUZ_CHECKTEST(LZ4_streamHC.internal_donotuse.dirty, "Context should be clean");
+ FUZ_CHECKTEST(LZ4_streamHC->internal_donotuse.dirty, "Context should be clean");
FUZ_DISPLAYTEST();
- LZ4_resetStreamHC_fast (&LZ4_streamHC, compressionLevel);
- LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC);
- ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
+ LZ4_resetStreamHC_fast (LZ4_streamHC, compressionLevel);
+ LZ4_attach_HC_dictionary(LZ4_streamHC, LZ4dictHC);
+ ret = LZ4_compress_HC_continue(LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDictCtx should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize);
/* note : context is no longer dirty after a failed compressed block */
FUZ_DISPLAYTEST();
- LZ4_resetStreamHC_fast (&LZ4_streamHC, compressionLevel);
- LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC);
- ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
+ LZ4_resetStreamHC_fast (LZ4_streamHC, compressionLevel);
+ LZ4_attach_HC_dictionary(LZ4_streamHC, LZ4dictHC);
+ ret = LZ4_compress_HC_continue(LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue using ExtDictCtx should work : enough size available within output buffer");
- FUZ_CHECKTEST(LZ4_streamHC.internal_donotuse.dirty, "Context should be clean");
+ FUZ_CHECKTEST(LZ4_streamHC->internal_donotuse.dirty, "Context should be clean");
FUZ_DISPLAYTEST();
- LZ4_resetStreamHC_fast (&LZ4_streamHC, compressionLevel);
- LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC);
- ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
+ LZ4_resetStreamHC_fast (LZ4_streamHC, compressionLevel);
+ LZ4_attach_HC_dictionary(LZ4_streamHC, LZ4dictHC);
+ ret = LZ4_compress_HC_continue(LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx and fast reset size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue using ExtDictCtx and fast reset should work : enough size available within output buffer");
- FUZ_CHECKTEST(LZ4_streamHC.internal_donotuse.dirty, "Context should be clean");
+ FUZ_CHECKTEST(LZ4_streamHC->internal_donotuse.dirty, "Context should be clean");
+
+ LZ4_freeStreamHC(LZ4_streamHC);
}
FUZ_DISPLAYTEST();
@@ -902,18 +906,19 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
{ U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0);
- if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
- FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
- }
+ if (crcCheck!=crcOrig) {
+ FUZ_findDiff(block, decodedBuffer);
+ EXIT_MSG("LZ4_decompress_safe_usingDict corrupted decoded data");
+ } }
/* Compress HC continue destSize */
FUZ_DISPLAYTEST();
{ int const availableSpace = (int)(FUZ_rand(&randState) % blockSize) + 5;
int consumedSize = blockSize;
FUZ_DISPLAYTEST();
- LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
- LZ4_setCompressionLevel(&LZ4dictHC, compressionLevel);
- blockContinueCompressedSize = LZ4_compress_HC_continue_destSize(&LZ4dictHC, block, compressedBuffer, &consumedSize, availableSpace);
+ LZ4_loadDictHC(LZ4dictHC, dict, dictSize);
+ LZ4_setCompressionLevel(LZ4dictHC, compressionLevel);
+ blockContinueCompressedSize = LZ4_compress_HC_continue_destSize(LZ4dictHC, block, compressedBuffer, &consumedSize, availableSpace);
DISPLAYLEVEL(5, " LZ4_compress_HC_continue_destSize : compressed %6i/%6i into %6i/%6i at cLevel=%i\n", consumedSize, blockSize, blockContinueCompressedSize, availableSpace, compressionLevel);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue_destSize failed");
FUZ_CHECKTEST(blockContinueCompressedSize > availableSpace, "LZ4_compress_HC_continue_destSize write overflow");
@@ -926,9 +931,10 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(decodedBuffer[consumedSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size")
{ U32 const crcSrc = XXH32(block, (size_t)consumedSize, 0);
U32 const crcDst = XXH32(decodedBuffer, (size_t)consumedSize, 0);
- if (crcSrc!=crcDst) FUZ_findDiff(block, decodedBuffer);
- FUZ_CHECKTEST(crcSrc!=crcDst, "LZ4_decompress_safe_usingDict corrupted decoded data");
- }
+ if (crcSrc!=crcDst) {
+ FUZ_findDiff(block, decodedBuffer);
+ EXIT_MSG("LZ4_decompress_safe_usingDict corrupted decoded data");
+ } }
}
/* ***** End of tests *** */
@@ -948,20 +954,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
printf("ratio with dict: %0.3f%%\n", (double)ccbytes/bytes*100);
/* release memory */
- {
-_exit:
- free(CNBuffer);
- free(compressedBuffer);
- free(decodedBuffer);
- FUZ_freeLowAddr(lowAddrBuffer, labSize);
- free(stateLZ4);
- free(stateLZ4HC);
- return result;
-
-_output_error:
- result = 1;
- goto _exit;
- }
+ free(CNBuffer);
+ free(compressedBuffer);
+ free(decodedBuffer);
+ FUZ_freeLowAddr(lowAddrBuffer, labSize);
+ LZ4_freeStreamHC(LZ4dictHC);
+ free(stateLZ4);
+ free(stateLZ4HC);
+ return result;
}
@@ -974,29 +974,55 @@ static void FUZ_unitTests(int compressionLevel)
const unsigned testNb = 0;
const unsigned seed = 0;
const unsigned cycleNb= 0;
- char testInput[testInputSize];
- char testCompressed[testCompressedSize];
- size_t const testVerifySize = testInputSize;
- char testVerify[testInputSize];
- char ringBuffer[ringBufferSize];
+ char* testInput = (char*)malloc(testInputSize);
+ char* testCompressed = (char*)malloc(testCompressedSize);
+ char* testVerify = (char*)malloc(testInputSize);
+ char ringBuffer[ringBufferSize] = {0};
U32 randState = 1;
/* Init */
+ if (!testInput || !testCompressed || !testVerify) {
+ EXIT_MSG("not enough memory for FUZ_unitTests");
+ }
FUZ_fillCompressibleNoiseBuffer(testInput, testInputSize, 0.50, &randState);
/* 32-bits address space overflow test */
FUZ_AddressOverflow();
+ /* Test decoding with empty input */
+ DISPLAYLEVEL(3, "LZ4_decompress_safe() with empty input");
+ LZ4_decompress_safe(testCompressed, testVerify, 0, testInputSize);
+
+ /* Test decoding with a one byte input */
+ DISPLAYLEVEL(3, "LZ4_decompress_safe() with one byte input");
+ { char const tmp = (char)0xFF;
+ LZ4_decompress_safe(&tmp, testVerify, 1, testInputSize);
+ }
+
+ /* Test decoding shortcut edge case */
+ DISPLAYLEVEL(3, "LZ4_decompress_safe() with shortcut edge case");
+ { char tmp[17];
+ /* 14 bytes of literals, followed by a 14 byte match.
+ * Should not read beyond the end of the buffer.
+ * See https://github.com/lz4/lz4/issues/508. */
+ *tmp = (char)0xEE;
+ memset(tmp + 1, 0, 14);
+ tmp[15] = 14;
+ tmp[16] = 0;
+ { int const r = LZ4_decompress_safe(tmp, testVerify, sizeof(tmp), testInputSize);
+ FUZ_CHECKTEST(r >= 0, "LZ4_decompress_safe() should fail");
+ } }
+
/* LZ4 streaming tests */
- { LZ4_stream_t* statePtr;
- LZ4_stream_t streamingState;
+ { LZ4_stream_t streamingState;
U64 crcOrig;
int result;
/* Allocation test */
- statePtr = LZ4_createStream();
- FUZ_CHECKTEST(statePtr==NULL, "LZ4_createStream() allocation failed");
- LZ4_freeStream(statePtr);
+ { LZ4_stream_t* const statePtr = LZ4_createStream();
+ FUZ_CHECKTEST(statePtr==NULL, "LZ4_createStream() allocation failed");
+ LZ4_freeStream(statePtr);
+ }
/* simple compression test */
crcOrig = XXH64(testInput, testCompressedSize, 0);
@@ -1021,7 +1047,6 @@ static void FUZ_unitTests(int compressionLevel)
U32 rNext = 0;
U32 dNext = 0;
const U32 dBufferSize = ringBufferSize + maxMessageSizeMask;
- int compressedSize;
XXH64_reset(&xxhOrig, 0);
XXH64_reset(&xxhNewSafe, 0);
@@ -1031,6 +1056,7 @@ static void FUZ_unitTests(int compressionLevel)
LZ4_setStreamDecode(&decodeStateFast, NULL, 0);
while (iNext + messageSize < testCompressedSize) {
+ int compressedSize;
XXH64_update(&xxhOrig, testInput + iNext, messageSize);
crcOrig = XXH64_digest(&xxhOrig);
@@ -1226,7 +1252,6 @@ static void FUZ_unitTests(int compressionLevel)
U32 rNext = 0;
U32 dNext = 0;
const U32 dBufferSize = ringBufferSize + maxMessageSizeMask;
- int compressedSize;
XXH64_reset(&xxhOrig, 0);
XXH64_reset(&xxhNewSafe, 0);
@@ -1236,6 +1261,7 @@ static void FUZ_unitTests(int compressionLevel)
LZ4_setStreamDecode(&decodeStateFast, NULL, 0);
while (iNext + messageSize < testCompressedSize) {
+ int compressedSize;
XXH64_update(&xxhOrig, testInput + iNext, messageSize);
crcOrig = XXH64_digest(&xxhOrig);
@@ -1289,7 +1315,7 @@ static void FUZ_unitTests(int compressionLevel)
int dNext = 0;
int compressedSize;
- assert((size_t)dBufferSize * 2 + 1 < testVerifySize); /* space used by ringBufferSafe and ringBufferFast */
+ assert((size_t)dBufferSize * 2 + 1 < testInputSize); /* space used by ringBufferSafe and ringBufferFast */
XXH64_reset(&xxhOrig, 0);
XXH64_reset(&xxhNewSafe, 0);
XXH64_reset(&xxhNewFast, 0);
@@ -1301,7 +1327,7 @@ static void FUZ_unitTests(int compressionLevel)
/* first block */
messageSize = BSIZE1; /* note : we cheat a bit here, in theory no message should be > maxMessageSize. We just want to fill the decoding ring buffer once. */
- XXH64_update(&xxhOrig, testInput + iNext, messageSize);
+ XXH64_update(&xxhOrig, testInput + iNext, (size_t)messageSize);
crcOrig = XXH64_digest(&xxhOrig);
compressedSize = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
@@ -1374,10 +1400,13 @@ static void FUZ_unitTests(int compressionLevel)
}
}
+ /* clean up */
+ free(testInput);
+ free(testCompressed);
+ free(testVerify);
+
printf("All unit tests completed successfully compressionLevel=%d \n", compressionLevel);
return;
-_output_error:
- exit(1);
}
generated by cgit v0.12 at 2024-06-01 19:39:38 (GMT)