summaryrefslogtreecommitdiffstats
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/Makefile62
-rw-r--r--lib/lz4.c69
-rw-r--r--lib/lz4.h124
-rw-r--r--lib/lz4frame.c1214
-rw-r--r--lib/lz4frame.h259
-rw-r--r--lib/lz4frame_static.h89
-rw-r--r--lib/lz4hc.c341
-rw-r--r--lib/lz4hc.h126
-rw-r--r--lib/lz4opt.h185
-rw-r--r--lib/xxhash.c27
10 files changed, 1517 insertions, 979 deletions
diff --git a/lib/Makefile b/lib/Makefile
index 2d9c8f3..9abb699 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -3,6 +3,8 @@
# Copyright (C) Yann Collet 2011-2016
# All rights reserved.
#
+# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
+#
# BSD license
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
@@ -40,14 +42,14 @@ LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
LIBVER := $(shell echo $(LIBVER_SCRIPT))
-BUILD_STATIC:= yes
+BUILD_STATIC:=yes
CPPFLAGS+= -DXXH_NAMESPACE=LZ4_
CFLAGS ?= -O3
-DEBUGFLAGS:=-g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+DEBUGFLAGS:= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
-Wpointer-arith -Wstrict-aliasing=1
-CFLAGS += $(MOREFLAGS)
+CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
@@ -58,7 +60,7 @@ ifeq ($(shell uname), Darwin)
SHARED_EXT = dylib
SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
- SONAME_FLAGS = -install_name $(PREFIX)/lib/liblz4.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
+ SONAME_FLAGS = -install_name $(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
else
SONAME_FLAGS = -Wl,-soname=liblz4.$(SHARED_EXT).$(LIBVER_MAJOR)
SHARED_EXT = so
@@ -68,12 +70,13 @@ endif
LIBLZ4 = liblz4.$(SHARED_EXT_VER)
+.PHONY: default
default: lib-release
-lib-release: liblz4.a liblz4
+lib-release: DEBUGFLAGS :=
+lib-release: lib
-lib: CFLAGS += $(DEBUGFLAGS)
-lib: lib-release
+lib: liblz4.a liblz4
all: lib
@@ -81,7 +84,7 @@ all32: CFLAGS+=-m32
all32: all
liblz4.a: *.c
-ifeq ($(BUILD_STATIC),yes)
+ifeq ($(BUILD_STATIC),yes) # can be disabled on command line
@echo compiling static library
@$(CC) $(CPPFLAGS) $(CFLAGS) -c $^
@$(AR) rcs $@ *.o
@@ -93,7 +96,7 @@ ifneq (,$(filter Windows%,$(OS)))
@$(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll\$@.dll
dlltool -D dll\liblz4.dll -d dll\liblz4.def -l dll\liblz4.lib
else
- @$(CC) $(FLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@
+ @$(CC) $(FLAGS) -shared $^ -fPIC -fvisibility=hidden $(SONAME_FLAGS) -o $@
@echo creating versioned links
@ln -sf $@ liblz4.$(SHARED_EXT_MAJOR)
@ln -sf $@ liblz4.$(SHARED_EXT)
@@ -112,16 +115,16 @@ clean:
#-----------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
-ifneq (,$(filter $(shell uname),SunOS))
-INSTALL ?= ginstall
-else
-INSTALL ?= install
-endif
-
-PREFIX ?= /usr/local
-DESTDIR ?=
-LIBDIR ?= $(PREFIX)/lib
-INCLUDEDIR ?= $(PREFIX)/include
+DESTDIR ?=
+# directory variables : GNU convention prefers lowercase
+# support both lower and uppercase (BSD), use uppercase in script
+prefix ?= /usr/local
+PREFIX ?= $(prefix)
+exec_prefix ?= $(PREFIX)
+libdir ?= $(exec_prefix)/lib
+LIBDIR ?= $(libdir)
+includedir ?= $(PREFIX)/include
+INCLUDEDIR ?= $(includedir)
ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly))
PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
@@ -129,8 +132,14 @@ else
PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig
endif
-INSTALL_LIB ?= $(INSTALL) -m 755
-INSTALL_DATA ?= $(INSTALL) -m 644
+ifneq (,$(filter $(shell uname),SunOS))
+INSTALL ?= ginstall
+else
+INSTALL ?= install
+endif
+
+INSTALL_PROGRAM ?= $(INSTALL)
+INSTALL_DATA ?= $(INSTALL) -m 644
liblz4.pc: liblz4.pc.in Makefile
@echo creating pkgconfig
@@ -141,20 +150,21 @@ liblz4.pc: liblz4.pc.in Makefile
$< >$@
install: lib liblz4.pc
- @$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/ $(DESTDIR)$(INCLUDEDIR)/
+ @$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/ $(DESTDIR)$(INCLUDEDIR)/ $(DESTDIR)$(LIBDIR)/
@$(INSTALL_DATA) liblz4.pc $(DESTDIR)$(PKGCONFIGDIR)/
@echo Installing libraries
ifeq ($(BUILD_STATIC),yes)
- @$(INSTALL_LIB) liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
+ @$(INSTALL_DATA) liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
+ @$(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(INCLUDEDIR)/lz4frame_static.h
endif
- @$(INSTALL_LIB) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)
+ @$(INSTALL_PROGRAM) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)
@ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
@ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
- @echo Installing includes
+ @echo Installing headers in $(INCLUDEDIR)
@$(INSTALL_DATA) lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h
@$(INSTALL_DATA) lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
@$(INSTALL_DATA) lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h
- @echo lz4 static and shared libraries installed
+ @echo lz4 libraries installed
uninstall:
@$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/liblz4.pc
diff --git a/lib/lz4.c b/lib/lz4.c
index 143c36e..41c0a28 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1,6 +1,6 @@
/*
LZ4 - Fast LZ compression algorithm
- Copyright (C) 2011-2016, Yann Collet.
+ Copyright (C) 2011-2017, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@@ -37,12 +37,12 @@
* Tuning parameters
**************************************/
/*
- * HEAPMODE :
+ * LZ4_HEAPMODE :
* Select how default compression functions will allocate memory for their hash table,
* in memory stack (0:default, fastest), or in memory heap (1:requires malloc()).
*/
-#ifndef HEAPMODE
-# define HEAPMODE 0
+#ifndef LZ4_HEAPMODE
+# define LZ4_HEAPMODE 0
#endif
/*
@@ -63,16 +63,15 @@
* Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
* Method 2 : direct access. This method is portable but violate C standard.
- * It can generate buggy code on targets which generate assembly depending on alignment.
+ * It can generate buggy code on targets which assembly generation depends on alignment.
* But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
* See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
* Prefer these methods in priority order (0 > 1 > 2)
*/
-#ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
+#ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally */
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
# define LZ4_FORCE_MEMORY_ACCESS 2
-# elif defined(__INTEL_COMPILER) || \
- (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
+# elif defined(__INTEL_COMPILER) || defined(__GNUC__)
# define LZ4_FORCE_MEMORY_ACCESS 1
# endif
#endif
@@ -97,20 +96,27 @@
* Compiler Options
**************************************/
#ifdef _MSC_VER /* Visual Studio */
-# define FORCE_INLINE static __forceinline
# include <intrin.h>
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */
-#else
-# if defined(__GNUC__) || defined(__clang__)
-# define FORCE_INLINE static inline __attribute__((always_inline))
-# elif defined(__cplusplus) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-# define FORCE_INLINE static inline
-# else
-# define FORCE_INLINE static
-# endif
#endif /* _MSC_VER */
+#ifndef FORCE_INLINE
+# ifdef _MSC_VER /* Visual Studio */
+# define FORCE_INLINE static __forceinline
+# else
+# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
+# ifdef __GNUC__
+# define FORCE_INLINE static inline __attribute__((always_inline))
+# else
+# define FORCE_INLINE static inline
+# endif
+# else
+# define FORCE_INLINE static
+# endif /* __STDC_VERSION__ */
+# endif /* _MSC_VER */
+#endif /* FORCE_INLINE */
+
#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__)
# define expect(expr,value) (__builtin_expect ((expr),(value)) )
#else
@@ -281,10 +287,22 @@ static const int LZ4_minLength = (MFLIMIT+1);
/*-************************************
-* Common Utils
+* Error detection
**************************************/
#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
+#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2)
+# include <stdio.h>
+# define DEBUGLOG(l, ...) { \
+ if (l<=LZ4_DEBUG) { \
+ fprintf(stderr, __FILE__ ": "); \
+ fprintf(stderr, __VA_ARGS__); \
+ fprintf(stderr, " \n"); \
+ } }
+#else
+# define DEBUGLOG(l, ...) {} /* disabled */
+#endif
+
/*-************************************
* Common functions
@@ -678,7 +696,7 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
{
-#if (HEAPMODE)
+#if (LZ4_HEAPMODE)
void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */
#else
LZ4_stream_t ctx;
@@ -687,7 +705,7 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
-#if (HEAPMODE)
+#if (LZ4_HEAPMODE)
FREEMEM(ctxPtr);
#endif
return result;
@@ -891,7 +909,7 @@ static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src,
int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize)
{
-#if (HEAPMODE)
+#if (LZ4_HEAPMODE)
LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */
#else
LZ4_stream_t ctxBody;
@@ -900,7 +918,7 @@ int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targe
int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize);
-#if (HEAPMODE)
+#if (LZ4_HEAPMODE)
FREEMEM(ctx);
#endif
return result;
@@ -927,6 +945,7 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
{
+ if (!LZ4_stream) return 0; /* support free on NULL */
FREEMEM(LZ4_stream);
return (0);
}
@@ -1266,11 +1285,6 @@ int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
/*===== streaming decompression functions =====*/
-/*
- * If you prefer dynamic allocation methods,
- * LZ4_createStreamDecode()
- * provides a pointer (void*) towards an initialized LZ4_streamDecode_t structure.
- */
LZ4_streamDecode_t* LZ4_createStreamDecode(void)
{
LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t));
@@ -1279,6 +1293,7 @@ LZ4_streamDecode_t* LZ4_createStreamDecode(void)
int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream)
{
+ if (!LZ4_stream) return 0; /* support free on NULL */
FREEMEM(LZ4_stream);
return 0;
}
diff --git a/lib/lz4.h b/lib/lz4.h
index 0aae19c..86ca0d5 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -1,7 +1,7 @@
/*
* LZ4 - Fast LZ compression algorithm
* Header File
- * Copyright (C) 2011-2016, Yann Collet.
+ * Copyright (C) 2011-2017, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@@ -32,13 +32,13 @@
- LZ4 homepage : http://www.lz4.org
- LZ4 source repository : https://github.com/lz4/lz4
*/
-#ifndef LZ4_H_2983827168210
-#define LZ4_H_2983827168210
-
#if defined (__cplusplus)
extern "C" {
#endif
+#ifndef LZ4_H_2983827168210
+#define LZ4_H_2983827168210
+
/* --- Dependency --- */
#include <stddef.h> /* size_t */
@@ -72,20 +72,24 @@ extern "C" {
/*
* LZ4_DLL_EXPORT :
* Enable exporting of functions when building a Windows DLL
+* LZ4LIB_API :
+* Control library symbols visibility.
*/
#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1)
# define LZ4LIB_API __declspec(dllexport)
#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1)
# define LZ4LIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#elif defined(__GNUC__) && (__GNUC__ >= 4)
+# define LZ4LIB_API __attribute__ ((__visibility__ ("default")))
#else
# define LZ4LIB_API
#endif
-/*========== Version =========== */
+/*------ Version ------*/
#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */
-#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */
-#define LZ4_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */
+#define LZ4_VERSION_MINOR 8 /* for new (non-breaking) interface capabilities */
+#define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */
#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
@@ -94,8 +98,8 @@ extern "C" {
#define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str)
#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION)
-LZ4LIB_API int LZ4_versionNumber (void);
-LZ4LIB_API const char* LZ4_versionString (void);
+LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; to be used when checking dll version */
+LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; to be used when checking dll version */
/*-************************************
@@ -108,8 +112,9 @@ LZ4LIB_API const char* LZ4_versionString (void);
* Reduced memory usage can improve speed, due to cache effect
* Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
*/
-#define LZ4_MEMORY_USAGE 14
-
+#ifndef LZ4_MEMORY_USAGE
+# define LZ4_MEMORY_USAGE 14
+#endif
/*-************************************
* Simple Functions
@@ -251,19 +256,19 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in
/*! LZ4_compress_fast_continue() :
* Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio.
- * Important : Previous data blocks are assumed to still be present and unmodified !
+ * Important : Previous data blocks are assumed to remain present and unmodified !
* 'dst' buffer must be already allocated.
- * If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
- * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero.
+ * If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
+ * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function @return==0.
+ * After an error, the stream status is invalid, it can only be reset or freed.
*/
-LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration);
+LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
/*! LZ4_saveDict() :
- * If previously compressed data block is not guaranteed to remain available at its memory location,
+ * If previously compressed data block is not guaranteed to remain available at its current memory location,
* save it into a safer place (char* safeBuffer).
- * Note : you don't need to call LZ4_loadDict() afterwards,
- * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue().
- * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
+ * Note : it's not necessary to call LZ4_loadDict() after LZ4_saveDict(), dictionary is immediately usable.
+ * @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
*/
LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize);
@@ -274,7 +279,8 @@ LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dict
************************************************/
typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* incomplete type (defined later) */
-/* creation / destruction of streaming decompression tracking structure */
+/*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() :
+ * creation / destruction of streaming decompression tracking structure */
LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void);
LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
@@ -285,22 +291,21 @@ LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_str
*/
LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
-/*!
-LZ4_decompress_*_continue() :
- These decoding functions allow decompression of multiple blocks in "streaming" mode.
- Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
- In the case of a ring buffers, decoding buffer must be either :
- - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
- In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
- - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
- maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
- In which case, encoding and decoding buffers do not need to be synchronized,
- and encoding ring buffer can have any size, including small ones ( < 64 KB).
- - _At least_ 64 KB + 8 bytes + maxBlockSize.
- In which case, encoding and decoding buffers do not need to be synchronized,
- and encoding ring buffer can have any size, including larger than decoding buffer.
- Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
- and indicate where it is saved using LZ4_setStreamDecode()
+/*! LZ4_decompress_*_continue() :
+ * These decoding functions allow decompression of multiple blocks in "streaming" mode.
+ * Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
+ * In the case of a ring buffers, decoding buffer must be either :
+ * - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
+ * In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
+ * - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
+ * maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
+ * In which case, encoding and decoding buffers do not need to be synchronized,
+ * and encoding ring buffer can have any size, including small ones ( < 64 KB).
+ * - _At least_ 64 KB + 8 bytes + maxBlockSize.
+ * In which case, encoding and decoding buffers do not need to be synchronized,
+ * and encoding ring buffer can have any size, including larger than decoding buffer.
+ * Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
+ * and indicate where it is saved using LZ4_setStreamDecode()
*/
LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize);
@@ -374,7 +379,7 @@ typedef struct {
* init this structure before first use.
* note : only use in association with static linking !
* this definition is not API/ABI safe,
- * and may change in a future version !
+ * it may change in a future version !
*/
#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4)
#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long))
@@ -400,11 +405,12 @@ union LZ4_streamDecode_u {
} ; /* previously typedef'd to LZ4_streamDecode_t */
-/*=************************************
+/*-************************************
* Obsolete Functions
**************************************/
-/* Deprecation warnings */
-/* Should these warnings be a problem,
+
+/*! Deprecation warnings
+ Should deprecation warnings be a problem,
it is generally possible to disable them,
typically with -Wno-deprecated-declarations for gcc
or _CRT_SECURE_NO_WARNINGS in Visual.
@@ -428,36 +434,30 @@ union LZ4_streamDecode_u {
#endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */
/* Obsolete compression functions */
-LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress (const char* source, char* dest, int sourceSize);
-LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize);
-LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
-LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
-LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
-LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress (const char* source, char* dest, int sourceSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
/* Obsolete decompression functions */
-/* These function names are completely deprecated and must no longer be used.
- They are only provided in lz4.c for compatibility with older programs.
- - LZ4_uncompress is the same as LZ4_decompress_fast
- - LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe
- These function prototypes are now disabled; uncomment them only if you really need them.
- It is highly recommended to stop using these prototypes and migrate to maintained ones */
-/* int LZ4_uncompress (const char* source, char* dest, int outputSize); */
-/* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast() instead") int LZ4_uncompress (const char* source, char* dest, int outputSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_safe() instead") int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
/* Obsolete streaming functions; use new streaming interface whenever possible */
-LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer);
-LZ4_DEPRECATED("use LZ4_createStream() instead") int LZ4_sizeofStreamState(void);
-LZ4_DEPRECATED("use LZ4_resetStream() instead") int LZ4_resetStreamState(void* state, char* inputBuffer);
-LZ4_DEPRECATED("use LZ4_saveDict() instead") char* LZ4_slideInputBuffer (void* state);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStream() instead") int LZ4_sizeofStreamState(void);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStream() instead") int LZ4_resetStreamState(void* state, char* inputBuffer);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_saveDict() instead") char* LZ4_slideInputBuffer (void* state);
/* Obsolete streaming decoding functions */
-LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
-LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
+
+#endif /* LZ4_H_2983827168210 */
#if defined (__cplusplus)
}
#endif
-
-#endif /* LZ4_H_2983827168210 */
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index a0a625b..3408708 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -61,15 +61,16 @@ You can contact the author at :
**************************************/
#include "lz4frame_static.h"
#include "lz4.h"
+#define LZ4_HC_STATIC_LINKING_ONLY
#include "lz4hc.h"
#define XXH_STATIC_LINKING_ONLY
#include "xxhash.h"
/*-************************************
-* Common Utils
+* Debug
**************************************/
-#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
+#define LZ4F_STATIC_ASSERT(c) { enum { LZ4F_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
/*-************************************
@@ -158,7 +159,7 @@ static void LZ4F_writeLE64 (void* dst, U64 value64)
#define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB
static const size_t minFHSize = 7;
-static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 15 */
+static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 19 */
static const size_t BHSize = 4;
@@ -170,6 +171,7 @@ typedef struct LZ4F_cctx_s
LZ4F_preferences_t prefs;
U32 version;
U32 cStage;
+ const LZ4F_CDict* cdict;
size_t maxBlockSize;
size_t maxBufferSize;
BYTE* tmpBuff;
@@ -178,7 +180,7 @@ typedef struct LZ4F_cctx_s
U64 totalInSize;
XXH32_state_t xxh;
void* lz4CtxPtr;
- U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */
+ U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */
} LZ4F_cctx_t;
@@ -209,12 +211,15 @@ LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult)
static LZ4F_errorCode_t err0r(LZ4F_errorCodes code)
{
- LZ4_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t)); /* A compilation error here means sizeof(ptrdiff_t) is not large enough */
+ /* A compilation error here means sizeof(ptrdiff_t) is not large enough */
+ LZ4F_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t));
return (LZ4F_errorCode_t)-(ptrdiff_t)code;
}
unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; }
+int LZ4F_compressionLevel_max(void) { return LZ4HC_CLEVEL_MAX; }
+
/*-************************************
* Private functions
@@ -241,7 +246,8 @@ static BYTE LZ4F_headerChecksum (const void* header, size_t length)
/*-************************************
* Simple-pass compression functions
**************************************/
-static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID, const size_t srcSize)
+static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID,
+ const size_t srcSize)
{
LZ4F_blockSizeID_t proposedBSID = LZ4F_max64KB;
size_t maxBlockSize = 64 KB;
@@ -254,21 +260,23 @@ static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSI
return requestedBSID;
}
-/* LZ4F_compressBound() :
- * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
- * prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario.
- * Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
- * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
+/*! LZ4F_compressBound_internal() :
+ * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
+ * prefsPtr is optional : if NULL is provided, preferences will be set to cover worst case scenario.
+ * @return is always the same for a srcSize and prefsPtr, so it can be relied upon to size reusable buffers.
+ * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
*/
-static size_t LZ4F_compressBound_internal(size_t srcSize, const LZ4F_preferences_t* preferencesPtr, size_t alreadyBuffered)
+static size_t LZ4F_compressBound_internal(size_t srcSize,
+ const LZ4F_preferences_t* preferencesPtr,
+ size_t alreadyBuffered)
{
LZ4F_preferences_t prefsNull;
memset(&prefsNull, 0, sizeof(prefsNull));
prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */
{ const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr;
U32 const flush = prefsPtr->autoFlush | (srcSize==0);
- LZ4F_blockSizeID_t const bid = prefsPtr->frameInfo.blockSizeID;
- size_t const blockSize = LZ4F_getBlockSize(bid);
+ LZ4F_blockSizeID_t const blockID = prefsPtr->frameInfo.blockSizeID;
+ size_t const blockSize = LZ4F_getBlockSize(blockID);
size_t const maxBuffered = blockSize - 1;
size_t const bufferedSize = MIN(alreadyBuffered, maxBuffered);
size_t const maxSrcSize = srcSize + bufferedSize;
@@ -277,17 +285,19 @@ static size_t LZ4F_compressBound_internal(size_t srcSize, const LZ4F_preferences
size_t const lastBlockSize = flush ? partialBlockSize : 0;
unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0);
- size_t const blockHeaderSize = 4; /* default, without block CRC option (which cannot be generated with current API) */
+ size_t const blockHeaderSize = 4;
+ size_t const blockCRCSize = 4 * prefsPtr->frameInfo.blockChecksumFlag;
size_t const frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4);
- return (blockHeaderSize * nbBlocks) + (blockSize * nbFullBlocks) + lastBlockSize + frameEnd;;
+ return ((blockHeaderSize + blockCRCSize) * nbBlocks) +
+ (blockSize * nbFullBlocks) + lastBlockSize + frameEnd;
}
}
size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
{
LZ4F_preferences_t prefs;
- size_t const headerSize = maxFHSize; /* max header size, including magic number and frame content size */
+ size_t const headerSize = maxFHSize; /* max header size, including optional fields */
if (preferencesPtr!=NULL) prefs = *preferencesPtr;
else memset(&prefs, 0, sizeof(prefs));
@@ -297,16 +307,19 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere
}
-/*! LZ4F_compressFrame() :
-* Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.0, in a single step.
-* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case.
-* You can get the minimum value of dstMaxSize by using LZ4F_compressFrameBound()
-* If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode)
-* The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will then be set to default.
-* The result of the function is the number of bytes written into dstBuffer.
-* The function outputs an error code if it fails (can be tested using LZ4F_isError())
-*/
-size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
+/*! LZ4F_compressFrame_usingCDict() :
+ * Compress srcBuffer using a dictionary, in a single step.
+ * cdict can be NULL, in which case, no dictionary is used.
+ * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
+ * The LZ4F_preferences_t structure is optional : you may provide NULL as argument,
+ * however, it's the only way to provide a dictID, so it's not recommended.
+ * @return : number of bytes written into dstBuffer,
+ * or an error code if it fails (can be tested using LZ4F_isError())
+ */
+size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity,
+ const void* srcBuffer, size_t srcSize,
+ const LZ4F_CDict* cdict,
+ const LZ4F_preferences_t* preferencesPtr)
{
LZ4F_cctx_t cctxI;
LZ4_stream_t lz4ctx;
@@ -316,11 +329,9 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu
BYTE* dstPtr = dstStart;
BYTE* const dstEnd = dstStart + dstCapacity;
- memset(&cctxI, 0, sizeof(cctxI)); /* works because no allocation */
- memset(&options, 0, sizeof(options));
-
+ memset(&cctxI, 0, sizeof(cctxI));
cctxI.version = LZ4F_VERSION;
- cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent allocation; works because autoflush==1 & stableSrc==1 */
+ cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */
if (preferencesPtr!=NULL)
prefs = *preferencesPtr;
@@ -329,22 +340,23 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu
if (prefs.frameInfo.contentSize != 0)
prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */
- if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
- cctxI.lz4CtxPtr = &lz4ctx;
- cctxI.lz4CtxLevel = 1;
- }
-
prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize);
prefs.autoFlush = 1;
if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID))
- prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* no need for linked blocks */
+ prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* only one block => no need for inter-block link */
+ if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
+ cctxI.lz4CtxPtr = &lz4ctx;
+ cctxI.lz4CtxLevel = 1;
+ } /* fast compression context pre-created on stack */
+
+ memset(&options, 0, sizeof(options));
options.stableSrc = 1;
- if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs))
+ if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs)) /* condition to guarantee success */
return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
- { size_t const headerSize = LZ4F_compressBegin(&cctxI, dstBuffer, dstCapacity, &prefs); /* write header */
+ { size_t const headerSize = LZ4F_compressBegin_usingCDict(&cctxI, dstBuffer, dstCapacity, cdict, &prefs); /* write header */
if (LZ4F_isError(headerSize)) return headerSize;
dstPtr += headerSize; /* header size */ }
@@ -356,24 +368,91 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu
if (LZ4F_isError(tailSize)) return tailSize;
dstPtr += tailSize; }
- if (prefs.compressionLevel >= LZ4HC_CLEVEL_MIN) /* no allocation done with lz4 fast */
+ if (prefs.compressionLevel >= LZ4HC_CLEVEL_MIN) /* Ctx allocation only for lz4hc */
FREEMEM(cctxI.lz4CtxPtr);
return (dstPtr - dstStart);
}
+/*! LZ4F_compressFrame() :
+ * Compress an entire srcBuffer into a valid LZ4 frame, in a single step.
+ * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
+ * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
+ * @return : number of bytes written into dstBuffer.
+ * or an error code if it fails (can be tested using LZ4F_isError())
+ */
+size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
+ const void* srcBuffer, size_t srcSize,
+ const LZ4F_preferences_t* preferencesPtr)
+{
+ return LZ4F_compressFrame_usingCDict(dstBuffer, dstCapacity,
+ srcBuffer, srcSize,
+ NULL, preferencesPtr);
+}
+
+
+/*-***************************************************
+* Dictionary compression
+*****************************************************/
+
+struct LZ4F_CDict_s {
+ void* dictContent;
+ LZ4_stream_t* fastCtx;
+ LZ4_streamHC_t* HCCtx;
+}; /* typedef'd to LZ4F_CDict within lz4frame_static.h */
+
+/*! LZ4F_createCDict() :
+ * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
+ * LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+ * LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
+ * `dictBuffer` can be released after LZ4F_CDict creation, since its content is copied within CDict
+ * @return : digested dictionary for compression, or NULL if failed */
+LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
+{
+ const char* dictStart = (const char*)dictBuffer;
+ LZ4F_CDict* cdict = (LZ4F_CDict*) malloc(sizeof(*cdict));
+ if (!cdict) return NULL;
+ if (dictSize > 64 KB) {
+ dictStart += dictSize - 64 KB;
+ dictSize = 64 KB;
+ }
+ cdict->dictContent = ALLOCATOR(dictSize);
+ cdict->fastCtx = LZ4_createStream();
+ cdict->HCCtx = LZ4_createStreamHC();
+ if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) {
+ LZ4F_freeCDict(cdict);
+ return NULL;
+ }
+ memcpy(cdict->dictContent, dictStart, dictSize);
+ LZ4_resetStream(cdict->fastCtx);
+ LZ4_loadDict (cdict->fastCtx, (const char*)cdict->dictContent, (int)dictSize);
+ LZ4_resetStreamHC(cdict->HCCtx, LZ4HC_CLEVEL_DEFAULT);
+ LZ4_loadDictHC(cdict->HCCtx, (const char*)cdict->dictContent, (int)dictSize);
+ return cdict;
+}
+
+void LZ4F_freeCDict(LZ4F_CDict* cdict)
+{
+ if (cdict==NULL) return; /* support free on NULL */
+ FREEMEM(cdict->dictContent);
+ LZ4_freeStream(cdict->fastCtx);
+ LZ4_freeStreamHC(cdict->HCCtx);
+ FREEMEM(cdict);
+}
+
+
/*-*********************************
* Advanced compression functions
***********************************/
/*! LZ4F_createCompressionContext() :
- * The first thing to do is to create a compressionContext object, which will be used in all compression operations.
- * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure.
- * The version provided MUST be LZ4F_VERSION. It is intended to track potential version differences between different binaries.
- * The function will provide a pointer to an allocated LZ4F_compressionContext_t object.
- * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation.
- * Object can release its memory using LZ4F_freeCompressionContext();
+ * The first thing to do is to create a compressionContext object, which will be used in all compression operations.
+ * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure.
+ * The version provided MUST be LZ4F_VERSION. It is intended to track potential incompatible differences between different binaries.
+ * The function will provide a pointer to an allocated LZ4F_compressionContext_t object.
+ * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation.
+ * Object can release its memory using LZ4F_freeCompressionContext();
*/
LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version)
{
@@ -381,7 +460,7 @@ LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_c
if (cctxPtr==NULL) return err0r(LZ4F_ERROR_allocation_failed);
cctxPtr->version = version;
- cctxPtr->cStage = 0; /* Next stage : write header */
+ cctxPtr->cStage = 0; /* Next stage : init stream */
*LZ4F_compressionContextPtr = (LZ4F_compressionContext_t)cctxPtr;
@@ -393,8 +472,8 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
{
LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)LZ4F_compressionContext;
- if (cctxPtr != NULL) { /* null pointers can be safely provided to this function, like free() */
- FREEMEM(cctxPtr->lz4CtxPtr);
+ if (cctxPtr != NULL) { /* support free on NULL */
+ FREEMEM(cctxPtr->lz4CtxPtr); /* works because LZ4_streamHC_t and LZ4_stream_t are simple POD types */
FREEMEM(cctxPtr->tmpBuff);
FREEMEM(LZ4F_compressionContext);
}
@@ -403,27 +482,28 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
}
-/*! LZ4F_compressBegin() :
- * will write the frame header into dstBuffer.
- * dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_HEADER_SIZE_MAX bytes.
+/*! LZ4F_compressBegin_usingCDict() :
+ * init streaming compression and writes frame header into dstBuffer.
+ * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes.
* @return : number of bytes written into dstBuffer for the header
* or an error code (can be tested using LZ4F_isError())
*/
-size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* preferencesPtr)
+size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
+ void* dstBuffer, size_t dstCapacity,
+ const LZ4F_CDict* cdict,
+ const LZ4F_preferences_t* preferencesPtr)
{
LZ4F_preferences_t prefNull;
BYTE* const dstStart = (BYTE*)dstBuffer;
BYTE* dstPtr = dstStart;
BYTE* headerStart;
- size_t requiredBuffSize;
if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
- if (cctxPtr->cStage != 0) return err0r(LZ4F_ERROR_GENERIC);
memset(&prefNull, 0, sizeof(prefNull));
if (preferencesPtr == NULL) preferencesPtr = &prefNull;
cctxPtr->prefs = *preferencesPtr;
- /* ctx Management */
+ /* Ctx Management */
{ U32 const tableID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */
if (cctxPtr->lz4CtxLevel < tableID) {
FREEMEM(cctxPtr->lz4CtxPtr);
@@ -431,31 +511,48 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit
cctxPtr->lz4CtxPtr = (void*)LZ4_createStream();
else
cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC();
+ if (cctxPtr->lz4CtxPtr == NULL) return err0r(LZ4F_ERROR_allocation_failed);
cctxPtr->lz4CtxLevel = tableID;
- }
- }
+ } }
/* Buffer Management */
- if (cctxPtr->prefs.frameInfo.blockSizeID == 0) cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT;
+ if (cctxPtr->prefs.frameInfo.blockSizeID == 0)
+ cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT;
cctxPtr->maxBlockSize = LZ4F_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID);
- requiredBuffSize = cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 128 KB);
- if (preferencesPtr->autoFlush)
- requiredBuffSize = (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 64 KB; /* just needs dict */
+ { size_t const requiredBuffSize = preferencesPtr->autoFlush ?
+ (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 64 KB : /* only needs windows size */
+ cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 128 KB);
- if (cctxPtr->maxBufferSize < requiredBuffSize) {
- cctxPtr->maxBufferSize = requiredBuffSize;
- FREEMEM(cctxPtr->tmpBuff);
- cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize);
- if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed);
- }
+ if (cctxPtr->maxBufferSize < requiredBuffSize) {
+ cctxPtr->maxBufferSize = 0;
+ FREEMEM(cctxPtr->tmpBuff);
+ cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize);
+ if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed);
+ cctxPtr->maxBufferSize = requiredBuffSize;
+ } }
cctxPtr->tmpIn = cctxPtr->tmpBuff;
cctxPtr->tmpInSize = 0;
XXH32_reset(&(cctxPtr->xxh), 0);
- if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
- LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr));
- else
- LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel);
+
+ /* context init */
+ cctxPtr->cdict = cdict;
+ if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) {
+ /* frame init only for blockLinked : blockIndependent will be init at each block */
+ if (cdict) {
+ if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
+ memcpy(cctxPtr->lz4CtxPtr, cdict->fastCtx, sizeof(*cdict->fastCtx));
+ } else {
+ memcpy(cctxPtr->lz4CtxPtr, cdict->HCCtx, sizeof(*cdict->HCCtx));
+ LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+ }
+ } else {
+ if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
+ LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr));
+ else
+ LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel);
+ }
+ }
/* Magic Number */
LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER);
@@ -464,9 +561,11 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit
/* FLG Byte */
*dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */
- + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) /* Block mode */
- + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) /* Frame checksum */
- + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3)); /* Frame content size */
+ + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5)
+ + ((cctxPtr->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4)
+ + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3)
+ + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2)
+ + (cctxPtr->prefs.frameInfo.dictID > 0) );
/* BD Byte */
*dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4);
/* Optional Frame content size field */
@@ -475,16 +574,36 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit
dstPtr += 8;
cctxPtr->totalInSize = 0;
}
- /* CRC Byte */
+ /* Optional dictionary ID field */
+ if (cctxPtr->prefs.frameInfo.dictID) {
+ LZ4F_writeLE32(dstPtr, cctxPtr->prefs.frameInfo.dictID);
+ dstPtr += 4;
+ }
+ /* Header CRC Byte */
*dstPtr = LZ4F_headerChecksum(headerStart, dstPtr - headerStart);
dstPtr++;
cctxPtr->cStage = 1; /* header written, now request input data block */
-
return (dstPtr - dstStart);
}
+/*! LZ4F_compressBegin() :
+ * init streaming compression and writes frame header into dstBuffer.
+ * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes.
+ * preferencesPtr can be NULL, in which case default parameters are selected.
+ * @return : number of bytes written into dstBuffer for the header
+ * or an error code (can be tested using LZ4F_isError())
+ */
+size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr,
+ void* dstBuffer, size_t dstCapacity,
+ const LZ4F_preferences_t* preferencesPtr)
+{
+ return LZ4F_compressBegin_usingCDict(cctxPtr, dstBuffer, dstCapacity,
+ NULL, preferencesPtr);
+}
+
+
/* LZ4F_compressBound() :
* @ return size of Dst buffer given a srcSize to handle worst case situations.
* The LZ4F_frameInfo_t structure is optional : if NULL, preferences will be set to cover worst case situations.
@@ -496,49 +615,75 @@ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesP
}
-typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level);
+typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level, const LZ4F_CDict* cdict);
-static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx, int level)
+
+/*! LZ4F_makeBlock():
+ * compress a single block, add header and checksum
+ * assumption : dst buffer capacity is >= srcSize */
+static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize,
+ compressFunc_t compress, void* lz4ctx, int level,
+ const LZ4F_CDict* cdict, LZ4F_blockChecksum_t crcFlag)
{
- /* compress a single block */
BYTE* const cSizePtr = (BYTE*)dst;
- U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1), level);
+ U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4),
+ (int)(srcSize), (int)(srcSize-1),
+ level, cdict);
LZ4F_writeLE32(cSizePtr, cSize);
if (cSize == 0) { /* compression failed */
cSize = (U32)srcSize;
LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG);
memcpy(cSizePtr+4, src, srcSize);
}
- return cSize + 4;
+ if (crcFlag) {
+ U32 const crc32 = XXH32(cSizePtr+4, cSize, 0); /* checksum of compressed data */
+ LZ4F_writeLE32(cSizePtr+4+cSize, crc32);
+ }
+ return 4 + cSize + ((U32)crcFlag)*4;
}
-static int LZ4F_localLZ4_compress_limitedOutput_withState(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level)
+static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
{
- (void) level;
- return LZ4_compress_fast_extState(ctx, src, dst, srcSize, dstCapacity, 1);
+ int const acceleration = (level < -1) ? -level : 1;
+ if (cdict) {
+ memcpy(ctx, cdict->fastCtx, sizeof(*cdict->fastCtx));
+ return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
+ }
+ return LZ4_compress_fast_extState(ctx, src, dst, srcSize, dstCapacity, acceleration);
+}
+
+static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
+{
+ int const acceleration = (level < -1) ? -level : 1;
+ (void)cdict; /* init once at beginning of frame */
+ return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
}
-static int LZ4F_localLZ4_compress_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level)
+static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
{
- (void) level;
- return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, 1);
+ if (cdict) {
+ memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx));
+ LZ4_setCompressionLevel((LZ4_streamHC_t*)ctx, level);
+ return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
+ }
+ return LZ4_compress_HC_extStateHC(ctx, src, dst, srcSize, dstCapacity, level);
}
-static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level)
+static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
{
- (void) level;
- return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstSize);
+ (void)level; (void)cdict; /* init once at beginning of frame */
+ return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
}
static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level)
{
if (level < LZ4HC_CLEVEL_MIN) {
- if (blockMode == LZ4F_blockIndependent) return LZ4F_localLZ4_compress_limitedOutput_withState;
- return LZ4F_localLZ4_compress_limitedOutput_continue;
+ if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlock;
+ return LZ4F_compressBlock_continue;
}
- if (blockMode == LZ4F_blockIndependent) return LZ4_compress_HC_extStateHC;
- return LZ4F_localLZ4_compressHC_limitedOutput_continue;
+ if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlockHC;
+ return LZ4F_compressBlockHC_continue;
}
static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
@@ -551,15 +696,16 @@ static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
/*! LZ4F_compressUpdate() :
-* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
-* The most important rule is that dstBuffer MUST be large enough (dstCapacity) to ensure compression completion even in worst case.
-* If this condition is not respected, LZ4F_compress() will fail (result is an errorCode)
-* You can get the minimum value of dstCapacity by using LZ4F_compressBound()
-* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
-* The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered.
-* The function outputs an error code if it fails (can be tested using LZ4F_isError())
-*/
-size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr)
+ * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
+ * dstBuffer MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
+ * LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
+ * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
+ * or an error code if it fails (which can be tested using LZ4F_isError())
+ */
+size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
+ void* dstBuffer, size_t dstCapacity,
+ const void* srcBuffer, size_t srcSize,
+ const LZ4F_compressOptions_t* compressOptionsPtr)
{
LZ4F_compressOptions_t cOptionsNull;
size_t const blockSize = cctxPtr->maxBlockSize;
@@ -591,7 +737,9 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci
memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy);
srcPtr += sizeToCopy;
- dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+ dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, blockSize,
+ compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
+ cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
cctxPtr->tmpInSize = 0;
@@ -599,16 +747,20 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci
}
while ((size_t)(srcEnd - srcPtr) >= blockSize) {
- /* compress full block */
+ /* compress full blocks */
lastBlockCompressed = fromSrcBuffer;
- dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+ dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, blockSize,
+ compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
+ cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
srcPtr += blockSize;
}
if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) {
/* compress remaining input < blockSize */
lastBlockCompressed = fromSrcBuffer;
- dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+ dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, srcEnd - srcPtr,
+ compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
+ cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
srcPtr = srcEnd;
}
@@ -617,7 +769,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci
if (compressOptionsPtr->stableSrc) {
cctxPtr->tmpIn = cctxPtr->tmpBuff;
} else {
- int realDictSize = LZ4F_localSaveDict(cctxPtr);
+ int const realDictSize = LZ4F_localSaveDict(cctxPtr);
if (realDictSize==0) return err0r(LZ4F_ERROR_GENERIC);
cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
}
@@ -627,7 +779,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci
if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) /* necessarily LZ4F_blockLinked && lastBlockCompressed==fromTmpBuffer */
&& !(cctxPtr->prefs.autoFlush))
{
- int realDictSize = LZ4F_localSaveDict(cctxPtr);
+ int const realDictSize = LZ4F_localSaveDict(cctxPtr);
cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
}
@@ -648,13 +800,13 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci
/*! LZ4F_flush() :
-* Should you need to create compressed data immediately, without waiting for a block to be filled,
-* you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext.
-* The result of the function is the number of bytes written into dstBuffer
-* (it can be zero, this means there was no data left within compressionContext)
-* The function outputs an error code if it fails (can be tested using LZ4F_isError())
-* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
-*/
+ * Should you need to create compressed data immediately, without waiting for a block to be filled,
+ * you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext.
+ * The result of the function is the number of bytes written into dstBuffer
+ * (it can be zero, this means there was no data left within compressionContext)
+ * The function outputs an error code if it fails (can be tested using LZ4F_isError())
+ * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
+ */
size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* compressOptionsPtr)
{
BYTE* const dstStart = (BYTE*)dstBuffer;
@@ -670,7 +822,9 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const
compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
/* compress tmp buffer */
- dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+ dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize,
+ compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
+ cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize;
cctxPtr->tmpInSize = 0;
@@ -685,14 +839,14 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const
/*! LZ4F_compressEnd() :
-* When you want to properly finish the compressed frame, just call LZ4F_compressEnd().
-* It will flush whatever data remained within compressionContext (like LZ4_flush())
-* but also properly finalize the frame, with an endMark and a checksum.
-* The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size))
-* The function outputs an error code if it fails (can be tested using LZ4F_isError())
-* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
-* compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same.
-*/
+ * When you want to properly finish the compressed frame, just call LZ4F_compressEnd().
+ * It will flush whatever data remained within compressionContext (like LZ4_flush())
+ * but also properly finalize the frame, with an endMark and a checksum.
+ * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size))
+ * The function outputs an error code if it fails (can be tested using LZ4F_isError())
+ * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
+ * compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same.
+ */
size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr)
{
BYTE* const dstStart = (BYTE*)dstBuffer;
@@ -727,10 +881,23 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize,
* Frame Decompression
*****************************************************/
+typedef enum {
+ dstage_getFrameHeader=0, dstage_storeFrameHeader,
+ dstage_init,
+ dstage_getBlockHeader, dstage_storeBlockHeader,
+ dstage_copyDirect, dstage_getBlockChecksum,
+ dstage_getCBlock, dstage_storeCBlock,
+ dstage_decodeCBlock, dstage_decodeCBlock_intoDst,
+ dstage_decodeCBlock_intoTmp, dstage_flushOut,
+ dstage_getSuffix, dstage_storeSuffix,
+ dstage_getSFrameSize, dstage_storeSFrameSize,
+ dstage_skipSkippable
+} dStage_t;
+
struct LZ4F_dctx_s {
LZ4F_frameInfo_t frameInfo;
U32 version;
- U32 dStage;
+ dStage_t dStage;
U64 frameRemainingSize;
size_t maxBlockSize;
size_t maxBufferSize;
@@ -738,40 +905,41 @@ struct LZ4F_dctx_s {
size_t tmpInSize;
size_t tmpInTarget;
BYTE* tmpOutBuffer;
- const BYTE* dict;
+ const BYTE* dict;
size_t dictSize;
BYTE* tmpOut;
size_t tmpOutSize;
size_t tmpOutStart;
XXH32_state_t xxh;
- BYTE header[16];
+ XXH32_state_t blockChecksum;
+ BYTE header[LZ4F_HEADER_SIZE_MAX];
}; /* typedef'd to LZ4F_dctx in lz4frame.h */
/*! LZ4F_createDecompressionContext() :
-* Create a decompressionContext object, which will track all decompression operations.
-* Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object.
-* Object can later be released using LZ4F_freeDecompressionContext().
-* @return : if != 0, there was an error during context creation.
-*/
+ * Create a decompressionContext object, which will track all decompression operations.
+ * Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object.
+ * Object can later be released using LZ4F_freeDecompressionContext().
+ * @return : if != 0, there was an error during context creation.
+ */
LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber)
{
- LZ4F_dctx* const dctxPtr = (LZ4F_dctx*)ALLOCATOR(sizeof(LZ4F_dctx));
- if (dctxPtr==NULL) return err0r(LZ4F_ERROR_GENERIC);
+ LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOCATOR(sizeof(LZ4F_dctx));
+ if (dctx==NULL) return err0r(LZ4F_ERROR_GENERIC);
- dctxPtr->version = versionNumber;
- *LZ4F_decompressionContextPtr = dctxPtr;
+ dctx->version = versionNumber;
+ *LZ4F_decompressionContextPtr = dctx;
return LZ4F_OK_NoError;
}
-LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctxPtr)
+LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx)
{
LZ4F_errorCode_t result = LZ4F_OK_NoError;
- if (dctxPtr != NULL) { /* can accept NULL input, like free() */
- result = (LZ4F_errorCode_t)dctxPtr->dStage;
- FREEMEM(dctxPtr->tmpIn);
- FREEMEM(dctxPtr->tmpOutBuffer);
- FREEMEM(dctxPtr);
+ if (dctx != NULL) { /* can accept NULL input, like free() */
+ result = (LZ4F_errorCode_t)dctx->dStage;
+ FREEMEM(dctx->tmpIn);
+ FREEMEM(dctx->tmpOutBuffer);
+ FREEMEM(dctx);
}
return result;
}
@@ -779,22 +947,18 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctxPtr)
/*==--- Streaming Decompression operations ---==*/
-typedef enum { dstage_getHeader=0, dstage_storeHeader,
- dstage_getCBlockSize, dstage_storeCBlockSize,
- dstage_copyDirect,
- dstage_getCBlock, dstage_storeCBlock,
- dstage_decodeCBlock, dstage_decodeCBlock_intoDst,
- dstage_decodeCBlock_intoTmp, dstage_flushOut,
- dstage_getSuffix, dstage_storeSuffix,
- dstage_getSFrameSize, dstage_storeSFrameSize,
- dstage_skipSkippable
-} dStage_t;
+void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx)
+{
+ dctx->dStage = dstage_getFrameHeader;
+ dctx->dict = NULL;
+ dctx->dictSize = 0;
+}
/*! LZ4F_headerSize() :
-* @return : size of frame header
-* or an error code, which can be tested using LZ4F_isError()
-*/
+ * @return : size of frame header
+ * or an error code, which can be tested using LZ4F_isError()
+ */
static size_t LZ4F_headerSize(const void* src, size_t srcSize)
{
/* minimal srcSize to determine header size */
@@ -804,244 +968,245 @@ static size_t LZ4F_headerSize(const void* src, size_t srcSize)
if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) return 8;
/* control magic number */
- if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER) return err0r(LZ4F_ERROR_frameType_unknown);
+ if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER)
+ return err0r(LZ4F_ERROR_frameType_unknown);
/* Frame Header Size */
{ BYTE const FLG = ((const BYTE*)src)[4];
U32 const contentSizeFlag = (FLG>>3) & _1BIT;
- return contentSizeFlag ? maxFHSize : minFHSize;
+ U32 const dictIDFlag = FLG & _1BIT;
+ return minFHSize + (contentSizeFlag*8) + (dictIDFlag*4);
}
}
/*! LZ4F_decodeHeader() :
- input : `src` points at the **beginning of the frame**
- output : set internal values of dctx, such as
- dctxPtr->frameInfo and dctxPtr->dStage.
- Also allocates internal buffers.
- @return : nb Bytes read from srcVoidPtr (necessarily <= srcSize)
- or an error code (testable with LZ4F_isError())
-*/
-static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcSize)
+ * input : `src` points at the **beginning of the frame**
+ * output : set internal values of dctx, such as
+ * dctx->frameInfo and dctx->dStage.
+ * Also allocates internal buffers.
+ * @return : nb Bytes read from src (necessarily <= srcSize)
+ * or an error code (testable with LZ4F_isError())
+ */
+static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize)
{
- BYTE FLG, BD;
- unsigned version, blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, blockSizeID;
+ unsigned blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, dictIDFlag, blockSizeID;
size_t frameHeaderSize;
const BYTE* srcPtr = (const BYTE*)src;
/* need to decode header to get frameInfo */
if (srcSize < minFHSize) return err0r(LZ4F_ERROR_frameHeader_incomplete); /* minimal frame header size */
- memset(&(dctxPtr->frameInfo), 0, sizeof(dctxPtr->frameInfo));
+ memset(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo));
/* special case : skippable frames */
if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) {
- dctxPtr->frameInfo.frameType = LZ4F_skippableFrame;
- if (src == (void*)(dctxPtr->header)) {
- dctxPtr->tmpInSize = srcSize;
- dctxPtr->tmpInTarget = 8;
- dctxPtr->dStage = dstage_storeSFrameSize;
+ dctx->frameInfo.frameType = LZ4F_skippableFrame;
+ if (src == (void*)(dctx->header)) {
+ dctx->tmpInSize = srcSize;
+ dctx->tmpInTarget = 8;
+ dctx->dStage = dstage_storeSFrameSize;
return srcSize;
} else {
- dctxPtr->dStage = dstage_getSFrameSize;
+ dctx->dStage = dstage_getSFrameSize;
return 4;
}
}
/* control magic number */
- if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) return err0r(LZ4F_ERROR_frameType_unknown);
- dctxPtr->frameInfo.frameType = LZ4F_frame;
+ if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER)
+ return err0r(LZ4F_ERROR_frameType_unknown);
+ dctx->frameInfo.frameType = LZ4F_frame;
/* Flags */
- FLG = srcPtr[4];
- version = (FLG>>6) & _2BITS;
- blockMode = (FLG>>5) & _1BIT;
- blockChecksumFlag = (FLG>>4) & _1BIT;
- contentSizeFlag = (FLG>>3) & _1BIT;
- contentChecksumFlag = (FLG>>2) & _1BIT;
+ { U32 const FLG = srcPtr[4];
+ U32 const version = (FLG>>6) & _2BITS;
+ blockChecksumFlag = (FLG>>4) & _1BIT;
+ blockMode = (FLG>>5) & _1BIT;
+ contentSizeFlag = (FLG>>3) & _1BIT;
+ contentChecksumFlag = (FLG>>2) & _1BIT;
+ dictIDFlag = FLG & _1BIT;
+ /* validate */
+ if (((FLG>>1)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bit */
+ if (version != 1) return err0r(LZ4F_ERROR_headerVersion_wrong); /* Version Number, only supported value */
+ }
/* Frame Header Size */
- frameHeaderSize = contentSizeFlag ? maxFHSize : minFHSize;
+ frameHeaderSize = minFHSize + (contentSizeFlag*8) + (dictIDFlag*4);
if (srcSize < frameHeaderSize) {
/* not enough input to fully decode frame header */
- if (srcPtr != dctxPtr->header)
- memcpy(dctxPtr->header, srcPtr, srcSize);
- dctxPtr->tmpInSize = srcSize;
- dctxPtr->tmpInTarget = frameHeaderSize;
- dctxPtr->dStage = dstage_storeHeader;
+ if (srcPtr != dctx->header)
+ memcpy(dctx->header, srcPtr, srcSize);
+ dctx->tmpInSize = srcSize;
+ dctx->tmpInTarget = frameHeaderSize;
+ dctx->dStage = dstage_storeFrameHeader;
return srcSize;
}
- BD = srcPtr[5];
- blockSizeID = (BD>>4) & _3BITS;
-
- /* validate */
- if (version != 1) return err0r(LZ4F_ERROR_headerVersion_wrong); /* Version Number, only supported value */
- if (blockChecksumFlag != 0) return err0r(LZ4F_ERROR_blockChecksum_unsupported); /* Not supported for the time being */
- if (((FLG>>0)&_2BITS) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bits */
- if (((BD>>7)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bit */
- if (blockSizeID < 4) return err0r(LZ4F_ERROR_maxBlockSize_invalid); /* 4-7 only supported values for the time being */
- if (((BD>>0)&_4BITS) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bits */
+ { U32 const BD = srcPtr[5];
+ blockSizeID = (BD>>4) & _3BITS;
+ /* validate */
+ if (((BD>>7)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bit */
+ if (blockSizeID < 4) return err0r(LZ4F_ERROR_maxBlockSize_invalid); /* 4-7 only supported values for the time being */
+ if (((BD>>0)&_4BITS) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bits */
+ }
/* check header */
- { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5);
- if (HC != srcPtr[frameHeaderSize-1]) return err0r(LZ4F_ERROR_headerChecksum_invalid); }
+ { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5);
+ if (HC != srcPtr[frameHeaderSize-1])
+ return err0r(LZ4F_ERROR_headerChecksum_invalid);
+ }
/* save */
- dctxPtr->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode;
- dctxPtr->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag;
- dctxPtr->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID;
- dctxPtr->maxBlockSize = LZ4F_getBlockSize(blockSizeID);
+ dctx->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode;
+ dctx->frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)blockChecksumFlag;
+ dctx->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag;
+ dctx->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID;
+ dctx->maxBlockSize = LZ4F_getBlockSize(blockSizeID);
if (contentSizeFlag)
- dctxPtr->frameRemainingSize = dctxPtr->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6);
-
- /* init */
- if (contentChecksumFlag) XXH32_reset(&(dctxPtr->xxh), 0);
-
- /* internal buffers allocation */
- { size_t const bufferNeeded = dctxPtr->maxBlockSize + ((dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB);
- if (bufferNeeded > dctxPtr->maxBufferSize) { /* tmp buffers too small */
- FREEMEM(dctxPtr->tmpIn);
- dctxPtr->tmpIn = (BYTE*)ALLOCATOR(dctxPtr->maxBlockSize);
- if (dctxPtr->tmpIn == NULL) return err0r(LZ4F_ERROR_allocation_failed);
- FREEMEM(dctxPtr->tmpOutBuffer);
- dctxPtr->maxBufferSize = 0;
- dctxPtr->tmpOutBuffer= (BYTE*)ALLOCATOR(bufferNeeded);
- if (dctxPtr->tmpOutBuffer== NULL) return err0r(LZ4F_ERROR_allocation_failed);
- dctxPtr->maxBufferSize = bufferNeeded;
- } }
- dctxPtr->tmpInSize = 0;
- dctxPtr->tmpInTarget = 0;
- dctxPtr->dict = dctxPtr->tmpOutBuffer;
- dctxPtr->dictSize = 0;
- dctxPtr->tmpOut = dctxPtr->tmpOutBuffer;
- dctxPtr->tmpOutStart = 0;
- dctxPtr->tmpOutSize = 0;
+ dctx->frameRemainingSize =
+ dctx->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6);
+ if (dictIDFlag)
+ dctx->frameInfo.dictID = LZ4F_readLE32(srcPtr + frameHeaderSize - 5);
- dctxPtr->dStage = dstage_getCBlockSize;
+ dctx->dStage = dstage_init;
return frameHeaderSize;
}
/*! LZ4F_getFrameInfo() :
-* Decodes frame header information, such as blockSize. Usage is optional.
-* The objective is to extract header information before receiving decompressed data, typically for allocation purposes.
-* LZ4F_getFrameInfo() can also be used *after* starting decompression, on a valid LZ4F_decompressionContext_t.
-* The number of bytes consumed from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
-* Decompression must resume from where it stopped (srcBuffer + *srcSizePtr)
-* @return : hint of the better `srcSize` to use for next call to LZ4F_decompress,
-* or an error code which can be tested using LZ4F_isError().
-*/
-LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctxPtr, LZ4F_frameInfo_t* frameInfoPtr,
+ * This function extracts frame parameters (max blockSize, frame checksum, etc.).
+ * Usage is optional. Objective is to provide relevant information for allocation purposes.
+ * This function works in 2 situations :
+ * - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process.
+ * Amount of input data provided must be large enough to successfully decode the frame header.
+ * A header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes. It's possible to provide more input data than this minimum.
+ * - After decoding has been started. In which case, no input is read, frame parameters are extracted from dctx.
+ * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
+ * Decompression must resume from (srcBuffer + *srcSizePtr).
+ * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
+ * or an error code which can be tested using LZ4F_isError()
+ * note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped.
+ * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
+ */
+LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoPtr,
const void* srcBuffer, size_t* srcSizePtr)
{
- if (dctxPtr->dStage > dstage_storeHeader) { /* note : requires dstage_* header related to be at beginning of enum */
+ if (dctx->dStage > dstage_storeFrameHeader) { /* assumption : dstage_* header enum at beginning of range */
/* frameInfo already decoded */
size_t o=0, i=0;
*srcSizePtr = 0;
- *frameInfoPtr = dctxPtr->frameInfo;
- return LZ4F_decompress(dctxPtr, NULL, &o, NULL, &i, NULL); /* returns : recommended nb of bytes for LZ4F_decompress() */
+ *frameInfoPtr = dctx->frameInfo;
+ /* returns : recommended nb of bytes for LZ4F_decompress() */
+ return LZ4F_decompress(dctx, NULL, &o, NULL, &i, NULL);
} else {
- size_t nextSrcSize, o=0;
- size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr);
- if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; }
- if (*srcSizePtr < hSize) { *srcSizePtr=0; return err0r(LZ4F_ERROR_frameHeader_incomplete); }
-
- *srcSizePtr = hSize;
- nextSrcSize = LZ4F_decompress(dctxPtr, NULL, &o, srcBuffer, srcSizePtr, NULL);
- if (dctxPtr->dStage <= dstage_storeHeader) return err0r(LZ4F_ERROR_frameHeader_incomplete); /* should not happen, already checked */
- *frameInfoPtr = dctxPtr->frameInfo;
- return nextSrcSize;
- }
-}
-
+ if (dctx->dStage == dstage_storeFrameHeader) {
+ /* frame decoding already started, in the middle of header => automatic fail */
+ *srcSizePtr = 0;
+ return err0r(LZ4F_ERROR_frameDecoding_alreadyStarted);
+ } else {
+ size_t decodeResult;
+ size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr);
+ if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; }
+ if (*srcSizePtr < hSize) {
+ *srcSizePtr=0;
+ return err0r(LZ4F_ERROR_frameHeader_incomplete);
+ }
-/* trivial redirector, for common prototype */
-static int LZ4F_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize)
-{
- (void)dictStart; (void)dictSize;
- return LZ4_decompress_safe (source, dest, compressedSize, maxDecompressedSize);
+ decodeResult = LZ4F_decodeHeader(dctx, srcBuffer, hSize);
+ if (LZ4F_isError(decodeResult)) {
+ *srcSizePtr = 0;
+ } else {
+ *srcSizePtr = decodeResult;
+ decodeResult = BHSize; /* block header size */
+ }
+ *frameInfoPtr = dctx->frameInfo;
+ return decodeResult;
+ } }
}
-static void LZ4F_updateDict(LZ4F_dctx* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp)
+/* LZ4F_updateDict() :
+ * only used for LZ4F_blockLinked mode */
+static void LZ4F_updateDict(LZ4F_dctx* dctx, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp)
{
- if (dctxPtr->dictSize==0)
- dctxPtr->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */
+ if (dctx->dictSize==0)
+ dctx->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */
- if (dctxPtr->dict + dctxPtr->dictSize == dstPtr) { /* dictionary continuity */
- dctxPtr->dictSize += dstSize;
+ if (dctx->dict + dctx->dictSize == dstPtr) { /* dictionary continuity */
+ dctx->dictSize += dstSize;
return;
}
if (dstPtr - dstPtr0 + dstSize >= 64 KB) { /* dstBuffer large enough to become dictionary */
- dctxPtr->dict = (const BYTE*)dstPtr0;
- dctxPtr->dictSize = dstPtr - dstPtr0 + dstSize;
+ dctx->dict = (const BYTE*)dstPtr0;
+ dctx->dictSize = dstPtr - dstPtr0 + dstSize;
return;
}
- if ((withinTmp) && (dctxPtr->dict == dctxPtr->tmpOutBuffer)) {
- /* assumption : dctxPtr->dict + dctxPtr->dictSize == dctxPtr->tmpOut + dctxPtr->tmpOutStart */
- dctxPtr->dictSize += dstSize;
+ if ((withinTmp) && (dctx->dict == dctx->tmpOutBuffer)) {
+ /* assumption : dctx->dict + dctx->dictSize == dctx->tmpOut + dctx->tmpOutStart */
+ dctx->dictSize += dstSize;
return;
}
if (withinTmp) { /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */
- size_t const preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer;
- size_t copySize = 64 KB - dctxPtr->tmpOutSize;
- const BYTE* const oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart;
- if (dctxPtr->tmpOutSize > 64 KB) copySize = 0;
+ size_t const preserveSize = dctx->tmpOut - dctx->tmpOutBuffer;
+ size_t copySize = 64 KB - dctx->tmpOutSize;
+ const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart;
+ if (dctx->tmpOutSize > 64 KB) copySize = 0;
if (copySize > preserveSize) copySize = preserveSize;
- memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize);
+ memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize);
- dctxPtr->dict = dctxPtr->tmpOutBuffer;
- dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart + dstSize;
+ dctx->dict = dctx->tmpOutBuffer;
+ dctx->dictSize = preserveSize + dctx->tmpOutStart + dstSize;
return;
}
- if (dctxPtr->dict == dctxPtr->tmpOutBuffer) { /* copy dst into tmp to complete dict */
- if (dctxPtr->dictSize + dstSize > dctxPtr->maxBufferSize) { /* tmp buffer not large enough */
+ if (dctx->dict == dctx->tmpOutBuffer) { /* copy dst into tmp to complete dict */
+ if (dctx->dictSize + dstSize > dctx->maxBufferSize) { /* tmp buffer not large enough */
size_t const preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */
- memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize);
- dctxPtr->dictSize = preserveSize;
+ memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize);
+ dctx->dictSize = preserveSize;
}
- memcpy(dctxPtr->tmpOutBuffer + dctxPtr->dictSize, dstPtr, dstSize);
- dctxPtr->dictSize += dstSize;
+ memcpy(dctx->tmpOutBuffer + dctx->dictSize, dstPtr, dstSize);
+ dctx->dictSize += dstSize;
return;
}
/* join dict & dest into tmp */
{ size_t preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */
- if (preserveSize > dctxPtr->dictSize) preserveSize = dctxPtr->dictSize;
- memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize);
- memcpy(dctxPtr->tmpOutBuffer + preserveSize, dstPtr, dstSize);
- dctxPtr->dict = dctxPtr->tmpOutBuffer;
- dctxPtr->dictSize = preserveSize + dstSize;
+ if (preserveSize > dctx->dictSize) preserveSize = dctx->dictSize;
+ memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize);
+ memcpy(dctx->tmpOutBuffer + preserveSize, dstPtr, dstSize);
+ dctx->dict = dctx->tmpOutBuffer;
+ dctx->dictSize = preserveSize + dstSize;
}
}
/*! LZ4F_decompress() :
-* Call this function repetitively to regenerate data compressed within srcBuffer.
-* The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
-*
-* The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
-*
-* The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
-* If the number of bytes read is < number of bytes provided, then the decompression operation is not complete.
-* Remaining data will have to be presented again in a subsequent invocation.
-*
-* The function result is an hint of the better srcSize to use for next call to LZ4F_decompress.
-* Basically, it's the size of the current (or remaining) compressed block + header of next block.
-* Respecting the hint provides some boost to performance, since it allows less buffer shuffling.
-* Note that this is just a hint, it's always possible to any srcSize value.
-* When a frame is fully decoded, @return will be 0.
-* If decompression failed, @return is an error code which can be tested using LZ4F_isError().
-*/
-size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
+ * Call this function repetitively to regenerate compressed data in srcBuffer.
+ * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer
+ * into dstBuffer of capacity *dstSizePtr.
+ *
+ * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
+ *
+ * The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
+ * If number of bytes read is < number of bytes provided, then decompression operation is not complete.
+ * Remaining data will have to be presented again in a subsequent invocation.
+ *
+ * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress.
+ * Schematically, it's the size of the current (or remaining) compressed block + header of next block.
+ * Respecting the hint provides a small boost to performance, since it allows less buffer shuffling.
+ * Note that this is just a hint, and it's always possible to any srcSize value.
+ * When a frame is fully decoded, @return will be 0.
+ * If decompression failed, @return is an error code which can be tested using LZ4F_isError().
+ */
+size_t LZ4F_decompress(LZ4F_dctx* dctx,
void* dstBuffer, size_t* dstSizePtr,
const void* srcBuffer, size_t* srcSizePtr,
const LZ4F_decompressOptions_t* decompressOptionsPtr)
@@ -1063,219 +1228,288 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
*srcSizePtr = 0;
*dstSizePtr = 0;
- /* programmed as a state machine */
+ /* behaves as a state machine */
while (doAnotherStage) {
- switch(dctxPtr->dStage)
+ switch(dctx->dStage)
{
- case dstage_getHeader:
+ case dstage_getFrameHeader:
if ((size_t)(srcEnd-srcPtr) >= maxFHSize) { /* enough to decode - shortcut */
- LZ4F_errorCode_t const hSize = LZ4F_decodeHeader(dctxPtr, srcPtr, srcEnd-srcPtr);
+ size_t const hSize = LZ4F_decodeHeader(dctx, srcPtr, srcEnd-srcPtr); /* will update dStage appropriately */
if (LZ4F_isError(hSize)) return hSize;
srcPtr += hSize;
break;
}
- dctxPtr->tmpInSize = 0;
- dctxPtr->tmpInTarget = minFHSize; /* minimum to attempt decode */
- dctxPtr->dStage = dstage_storeHeader;
- /* pass-through */
-
- case dstage_storeHeader:
- { size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize;
- if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr;
- memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
- dctxPtr->tmpInSize += sizeToCopy;
+ dctx->tmpInSize = 0;
+ if (srcEnd-srcPtr == 0) return minFHSize; /* 0-size input */
+ dctx->tmpInTarget = minFHSize; /* minimum to attempt decode */
+ dctx->dStage = dstage_storeFrameHeader;
+ /* fall-through */
+
+ case dstage_storeFrameHeader:
+ { size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, (size_t)(srcEnd - srcPtr));
+ memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);
+ dctx->tmpInSize += sizeToCopy;
srcPtr += sizeToCopy;
- if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) {
- nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + BHSize; /* rest of header + nextBlockHeader */
+ if (dctx->tmpInSize < dctx->tmpInTarget) {
+ nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize; /* rest of header + nextBlockHeader */
doAnotherStage = 0; /* not enough src data, ask for some more */
break;
}
- { LZ4F_errorCode_t const hSize = LZ4F_decodeHeader(dctxPtr, dctxPtr->header, dctxPtr->tmpInTarget);
+ { size_t const hSize = LZ4F_decodeHeader(dctx, dctx->header, dctx->tmpInTarget); /* will update dStage appropriately */
if (LZ4F_isError(hSize)) return hSize;
}
break;
}
- case dstage_getCBlockSize:
+ case dstage_init:
+ if (dctx->frameInfo.contentChecksumFlag) XXH32_reset(&(dctx->xxh), 0);
+ /* internal buffers allocation */
+ { size_t const bufferNeeded = dctx->maxBlockSize + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB) + 4 /* block checksum */;
+ if (bufferNeeded > dctx->maxBufferSize) { /* tmp buffers too small */
+ dctx->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/
+ FREEMEM(dctx->tmpIn);
+ dctx->tmpIn = (BYTE*)ALLOCATOR(dctx->maxBlockSize);
+ if (dctx->tmpIn == NULL) return err0r(LZ4F_ERROR_allocation_failed);
+ FREEMEM(dctx->tmpOutBuffer);
+ dctx->tmpOutBuffer= (BYTE*)ALLOCATOR(bufferNeeded);
+ if (dctx->tmpOutBuffer== NULL) return err0r(LZ4F_ERROR_allocation_failed);
+ dctx->maxBufferSize = bufferNeeded;
+ } }
+ dctx->tmpInSize = 0;
+ dctx->tmpInTarget = 0;
+ dctx->tmpOut = dctx->tmpOutBuffer;
+ dctx->tmpOutStart = 0;
+ dctx->tmpOutSize = 0;
+
+ dctx->dStage = dstage_getBlockHeader;
+ /* fall-through */
+
+ case dstage_getBlockHeader:
if ((size_t)(srcEnd - srcPtr) >= BHSize) {
selectedIn = srcPtr;
srcPtr += BHSize;
} else {
/* not enough input to read cBlockSize field */
- dctxPtr->tmpInSize = 0;
- dctxPtr->dStage = dstage_storeCBlockSize;
+ dctx->tmpInSize = 0;
+ dctx->dStage = dstage_storeBlockHeader;
}
- if (dctxPtr->dStage == dstage_storeCBlockSize) /* can be skipped */
- case dstage_storeCBlockSize:
- { size_t sizeToCopy = BHSize - dctxPtr->tmpInSize;
+ if (dctx->dStage == dstage_storeBlockHeader) /* can be skipped */
+ case dstage_storeBlockHeader:
+ { size_t sizeToCopy = BHSize - dctx->tmpInSize;
if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr;
- memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
+ memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy);
srcPtr += sizeToCopy;
- dctxPtr->tmpInSize += sizeToCopy;
- if (dctxPtr->tmpInSize < BHSize) { /* not enough input to get full cBlockSize; wait for more */
- nextSrcSizeHint = BHSize - dctxPtr->tmpInSize;
+ dctx->tmpInSize += sizeToCopy;
+ if (dctx->tmpInSize < BHSize) { /* not enough input for cBlockSize */
+ nextSrcSizeHint = BHSize - dctx->tmpInSize;
doAnotherStage = 0;
break;
}
- selectedIn = dctxPtr->tmpIn;
+ selectedIn = dctx->tmpIn;
}
- /* case dstage_decodeCBlockSize: */ /* no more direct access, to prevent scan-build warning */
+ /* decode block header */
{ size_t const nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU;
- if (nextCBlockSize==0) { /* frameEnd signal, no more CBlock */
- dctxPtr->dStage = dstage_getSuffix;
+ size_t const crcSize = dctx->frameInfo.blockChecksumFlag * 4;
+ if (nextCBlockSize==0) { /* frameEnd signal, no more block */
+ dctx->dStage = dstage_getSuffix;
break;
}
- if (nextCBlockSize > dctxPtr->maxBlockSize) return err0r(LZ4F_ERROR_GENERIC); /* invalid cBlockSize */
- dctxPtr->tmpInTarget = nextCBlockSize;
+ if (nextCBlockSize > dctx->maxBlockSize)
+ return err0r(LZ4F_ERROR_maxBlockSize_invalid);
if (LZ4F_readLE32(selectedIn) & LZ4F_BLOCKUNCOMPRESSED_FLAG) {
- dctxPtr->dStage = dstage_copyDirect;
+ /* next block is uncompressed */
+ dctx->tmpInTarget = nextCBlockSize;
+ if (dctx->frameInfo.blockChecksumFlag) {
+ XXH32_reset(&dctx->blockChecksum, 0);
+ }
+ dctx->dStage = dstage_copyDirect;
break;
}
- dctxPtr->dStage = dstage_getCBlock;
+ /* next block is a compressed block */
+ dctx->tmpInTarget = nextCBlockSize + crcSize;
+ dctx->dStage = dstage_getCBlock;
if (dstPtr==dstEnd) {
- nextSrcSizeHint = nextCBlockSize + BHSize;
+ nextSrcSizeHint = nextCBlockSize + crcSize + BHSize;
doAnotherStage = 0;
}
break;
}
case dstage_copyDirect: /* uncompressed block */
- { size_t sizeToCopy = dctxPtr->tmpInTarget;
- if ((size_t)(srcEnd-srcPtr) < sizeToCopy) sizeToCopy = srcEnd - srcPtr; /* not enough input to read full block */
- if ((size_t)(dstEnd-dstPtr) < sizeToCopy) sizeToCopy = dstEnd - dstPtr;
+ { size_t const minBuffSize = MIN((size_t)(srcEnd-srcPtr), (size_t)(dstEnd-dstPtr));
+ size_t const sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize);
memcpy(dstPtr, srcPtr, sizeToCopy);
- if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), srcPtr, sizeToCopy);
- if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= sizeToCopy;
+ if (dctx->frameInfo.blockChecksumFlag) {
+ XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy);
+ }
+ if (dctx->frameInfo.contentChecksumFlag)
+ XXH32_update(&dctx->xxh, srcPtr, sizeToCopy);
+ if (dctx->frameInfo.contentSize)
+ dctx->frameRemainingSize -= sizeToCopy;
- /* dictionary management */
- if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked)
- LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 0);
+ /* history management (linked blocks only)*/
+ if (dctx->frameInfo.blockMode == LZ4F_blockLinked)
+ LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 0);
srcPtr += sizeToCopy;
dstPtr += sizeToCopy;
- if (sizeToCopy == dctxPtr->tmpInTarget) { /* all copied */
- dctxPtr->dStage = dstage_getCBlockSize;
+ if (sizeToCopy == dctx->tmpInTarget) { /* all done */
+ if (dctx->frameInfo.blockChecksumFlag) {
+ dctx->tmpInSize = 0;
+ dctx->dStage = dstage_getBlockChecksum;
+ } else
+ dctx->dStage = dstage_getBlockHeader; /* new block */
break;
}
- dctxPtr->tmpInTarget -= sizeToCopy; /* still need to copy more */
- nextSrcSizeHint = dctxPtr->tmpInTarget + BHSize;
+ dctx->tmpInTarget -= sizeToCopy; /* need to copy more */
+ nextSrcSizeHint = dctx->tmpInTarget +
+ + dctx->frameInfo.contentChecksumFlag * 4 /* block checksum */
+ + BHSize /* next header size */;
doAnotherStage = 0;
break;
}
+ /* check block checksum for recently transferred uncompressed block */
+ case dstage_getBlockChecksum:
+ { const void* crcSrc;
+ if ((srcEnd-srcPtr >= 4) && (dctx->tmpInSize==0)) {
+ crcSrc = srcPtr;
+ srcPtr += 4;
+ } else {
+ size_t const stillToCopy = 4 - dctx->tmpInSize;
+ size_t const sizeToCopy = MIN(stillToCopy, (size_t)(srcEnd-srcPtr));
+ memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);
+ dctx->tmpInSize += sizeToCopy;
+ srcPtr += sizeToCopy;
+ if (dctx->tmpInSize < 4) { /* all input consumed */
+ doAnotherStage = 0;
+ break;
+ }
+ crcSrc = dctx->header;
+ }
+ { U32 const readCRC = LZ4F_readLE32(crcSrc);
+ U32 const calcCRC = XXH32_digest(&dctx->blockChecksum);
+ if (readCRC != calcCRC)
+ return err0r(LZ4F_ERROR_blockChecksum_invalid);
+ }
+ }
+ dctx->dStage = dstage_getBlockHeader; /* new block */
+ break;
+
case dstage_getCBlock: /* entry from dstage_decodeCBlockSize */
- if ((size_t)(srcEnd-srcPtr) < dctxPtr->tmpInTarget) {
- dctxPtr->tmpInSize = 0;
- dctxPtr->dStage = dstage_storeCBlock;
+ if ((size_t)(srcEnd-srcPtr) < dctx->tmpInTarget) {
+ dctx->tmpInSize = 0;
+ dctx->dStage = dstage_storeCBlock;
break;
}
+ /* input large enough to read full block directly */
selectedIn = srcPtr;
- srcPtr += dctxPtr->tmpInTarget;
- dctxPtr->dStage = dstage_decodeCBlock;
+ srcPtr += dctx->tmpInTarget;
+ dctx->dStage = dstage_decodeCBlock;
break;
case dstage_storeCBlock:
- { size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize;
- if (sizeToCopy > (size_t)(srcEnd-srcPtr)) sizeToCopy = srcEnd-srcPtr;
- memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
- dctxPtr->tmpInSize += sizeToCopy;
+ { size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, (size_t)(srcEnd-srcPtr));
+ memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy);
+ dctx->tmpInSize += sizeToCopy;
srcPtr += sizeToCopy;
- if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) { /* need more input */
- nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + BHSize;
+ if (dctx->tmpInSize < dctx->tmpInTarget) { /* need more input */
+ nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize;
doAnotherStage=0;
break;
}
- selectedIn = dctxPtr->tmpIn;
- dctxPtr->dStage = dstage_decodeCBlock;
- /* pass-through */
+ selectedIn = dctx->tmpIn;
+ dctx->dStage = dstage_decodeCBlock;
}
+ /* fall-through */
+ /* At this stage, input is large enough to decode a block */
case dstage_decodeCBlock:
- if ((size_t)(dstEnd-dstPtr) < dctxPtr->maxBlockSize) /* not enough place into dst : decode into tmpOut */
- dctxPtr->dStage = dstage_decodeCBlock_intoTmp;
+ if (dctx->frameInfo.blockChecksumFlag) {
+ dctx->tmpInTarget -= 4;
+ { U32 const readBlockCrc = LZ4F_readLE32(selectedIn + dctx->tmpInTarget);
+ U32 const calcBlockCrc = XXH32(selectedIn, dctx->tmpInTarget, 0);
+ if (readBlockCrc != calcBlockCrc)
+ return err0r(LZ4F_ERROR_blockChecksum_invalid);
+ } }
+ if ((size_t)(dstEnd-dstPtr) < dctx->maxBlockSize) /* not enough place into dst : decode into tmpOut */
+ dctx->dStage = dstage_decodeCBlock_intoTmp;
else
- dctxPtr->dStage = dstage_decodeCBlock_intoDst;
+ dctx->dStage = dstage_decodeCBlock_intoDst;
break;
case dstage_decodeCBlock_intoDst:
- { int (*decoder)(const char*, char*, int, int, const char*, int);
- int decodedSize;
-
- if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked)
- decoder = LZ4_decompress_safe_usingDict;
- else
- decoder = LZ4F_decompress_safe;
-
- decodedSize = decoder((const char*)selectedIn, (char*)dstPtr, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize);
+ { int const decodedSize = LZ4_decompress_safe_usingDict(
+ (const char*)selectedIn, (char*)dstPtr,
+ (int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
+ (const char*)dctx->dict, (int)dctx->dictSize);
if (decodedSize < 0) return err0r(LZ4F_ERROR_GENERIC); /* decompression failed */
- if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize);
- if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize;
+ if (dctx->frameInfo.contentChecksumFlag)
+ XXH32_update(&(dctx->xxh), dstPtr, decodedSize);
+ if (dctx->frameInfo.contentSize)
+ dctx->frameRemainingSize -= decodedSize;
/* dictionary management */
- if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked)
- LZ4F_updateDict(dctxPtr, dstPtr, decodedSize, dstStart, 0);
+ if (dctx->frameInfo.blockMode==LZ4F_blockLinked)
+ LZ4F_updateDict(dctx, dstPtr, decodedSize, dstStart, 0);
dstPtr += decodedSize;
- dctxPtr->dStage = dstage_getCBlockSize;
+ dctx->dStage = dstage_getBlockHeader;
break;
}
case dstage_decodeCBlock_intoTmp:
/* not enough place into dst : decode into tmpOut */
- { int (*decoder)(const char*, char*, int, int, const char*, int);
- int decodedSize;
-
- if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked)
- decoder = LZ4_decompress_safe_usingDict;
- else
- decoder = LZ4F_decompress_safe;
-
- /* ensure enough place for tmpOut */
- if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked) {
- if (dctxPtr->dict == dctxPtr->tmpOutBuffer) {
- if (dctxPtr->dictSize > 128 KB) {
- memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - 64 KB, 64 KB);
- dctxPtr->dictSize = 64 KB;
- }
- dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + dctxPtr->dictSize;
- } else { /* dict not within tmp */
- size_t reservedDictSpace = dctxPtr->dictSize;
- if (reservedDictSpace > 64 KB) reservedDictSpace = 64 KB;
- dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + reservedDictSpace;
+
+ /* ensure enough place for tmpOut */
+ if (dctx->frameInfo.blockMode == LZ4F_blockLinked) {
+ if (dctx->dict == dctx->tmpOutBuffer) {
+ if (dctx->dictSize > 128 KB) {
+ memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - 64 KB, 64 KB);
+ dctx->dictSize = 64 KB;
}
+ dctx->tmpOut = dctx->tmpOutBuffer + dctx->dictSize;
+ } else { /* dict not within tmp */
+ size_t const reservedDictSpace = MIN(dctx->dictSize, 64 KB);
+ dctx->tmpOut = dctx->tmpOutBuffer + reservedDictSpace;
}
+ }
- /* Decode */
- decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize);
- if (decodedSize < 0) return err0r(LZ4F_ERROR_decompressionFailed); /* decompression failed */
- if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize);
- if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize;
- dctxPtr->tmpOutSize = decodedSize;
- dctxPtr->tmpOutStart = 0;
- dctxPtr->dStage = dstage_flushOut;
- break;
+ /* Decode block */
+ { int const decodedSize = LZ4_decompress_safe_usingDict(
+ (const char*)selectedIn, (char*)dctx->tmpOut,
+ (int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
+ (const char*)dctx->dict, (int)dctx->dictSize);
+ if (decodedSize < 0) /* decompression failed */
+ return err0r(LZ4F_ERROR_decompressionFailed);
+ if (dctx->frameInfo.contentChecksumFlag)
+ XXH32_update(&(dctx->xxh), dctx->tmpOut, decodedSize);
+ if (dctx->frameInfo.contentSize)
+ dctx->frameRemainingSize -= decodedSize;
+ dctx->tmpOutSize = decodedSize;
+ dctx->tmpOutStart = 0;
+ dctx->dStage = dstage_flushOut;
}
+ /* fall-through */
case dstage_flushOut: /* flush decoded data from tmpOut to dstBuffer */
- { size_t sizeToCopy = dctxPtr->tmpOutSize - dctxPtr->tmpOutStart;
- if (sizeToCopy > (size_t)(dstEnd-dstPtr)) sizeToCopy = dstEnd-dstPtr;
- memcpy(dstPtr, dctxPtr->tmpOut + dctxPtr->tmpOutStart, sizeToCopy);
+ { size_t const sizeToCopy = MIN(dctx->tmpOutSize - dctx->tmpOutStart, (size_t)(dstEnd-dstPtr));
+ memcpy(dstPtr, dctx->tmpOut + dctx->tmpOutStart, sizeToCopy);
/* dictionary management */
- if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked)
- LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 1);
+ if (dctx->frameInfo.blockMode==LZ4F_blockLinked)
+ LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 1);
- dctxPtr->tmpOutStart += sizeToCopy;
+ dctx->tmpOutStart += sizeToCopy;
dstPtr += sizeToCopy;
- /* end of flush ? */
- if (dctxPtr->tmpOutStart == dctxPtr->tmpOutSize) {
- dctxPtr->dStage = dstage_getCBlockSize;
+ if (dctx->tmpOutStart == dctx->tmpOutSize) { /* all flushed */
+ dctx->dStage = dstage_getBlockHeader; /* get next block */
break;
}
nextSrcSizeHint = BHSize;
@@ -1284,45 +1518,47 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
}
case dstage_getSuffix:
- { size_t const suffixSize = dctxPtr->frameInfo.contentChecksumFlag * 4;
- if (dctxPtr->frameRemainingSize) return err0r(LZ4F_ERROR_frameSize_wrong); /* incorrect frame size decoded */
+ { size_t const suffixSize = dctx->frameInfo.contentChecksumFlag * 4;
+ if (dctx->frameRemainingSize)
+ return err0r(LZ4F_ERROR_frameSize_wrong); /* incorrect frame size decoded */
if (suffixSize == 0) { /* frame completed */
nextSrcSizeHint = 0;
- dctxPtr->dStage = dstage_getHeader;
+ LZ4F_resetDecompressionContext(dctx);
doAnotherStage = 0;
break;
}
if ((srcEnd - srcPtr) < 4) { /* not enough size for entire CRC */
- dctxPtr->tmpInSize = 0;
- dctxPtr->dStage = dstage_storeSuffix;
+ dctx->tmpInSize = 0;
+ dctx->dStage = dstage_storeSuffix;
} else {
selectedIn = srcPtr;
srcPtr += 4;
}
}
- if (dctxPtr->dStage == dstage_storeSuffix) /* can be skipped */
+ if (dctx->dStage == dstage_storeSuffix) /* can be skipped */
case dstage_storeSuffix:
{
- size_t sizeToCopy = 4 - dctxPtr->tmpInSize;
+ size_t sizeToCopy = 4 - dctx->tmpInSize;
if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr;
- memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
+ memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy);
srcPtr += sizeToCopy;
- dctxPtr->tmpInSize += sizeToCopy;
- if (dctxPtr->tmpInSize < 4) { /* not enough input to read complete suffix */
- nextSrcSizeHint = 4 - dctxPtr->tmpInSize;
+ dctx->tmpInSize += sizeToCopy;
+ if (dctx->tmpInSize < 4) { /* not enough input to read complete suffix */
+ nextSrcSizeHint = 4 - dctx->tmpInSize;
doAnotherStage=0;
break;
}
- selectedIn = dctxPtr->tmpIn;
+ selectedIn = dctx->tmpIn;
}
- /* case dstage_checkSuffix: */ /* no direct call, to avoid scan-build warning */
+ /* case dstage_checkSuffix: */ /* no direct call, avoid scan-build warning */
{ U32 const readCRC = LZ4F_readLE32(selectedIn);
- U32 const resultCRC = XXH32_digest(&(dctxPtr->xxh));
- if (readCRC != resultCRC) return err0r(LZ4F_ERROR_contentChecksum_invalid);
+ U32 const resultCRC = XXH32_digest(&(dctx->xxh));
+ if (readCRC != resultCRC)
+ return err0r(LZ4F_ERROR_contentChecksum_invalid);
nextSrcSizeHint = 0;
- dctxPtr->dStage = dstage_getHeader;
+ LZ4F_resetDecompressionContext(dctx);
doAnotherStage = 0;
break;
}
@@ -1333,77 +1569,77 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
srcPtr += 4;
} else {
/* not enough input to read cBlockSize field */
- dctxPtr->tmpInSize = 4;
- dctxPtr->tmpInTarget = 8;
- dctxPtr->dStage = dstage_storeSFrameSize;
+ dctx->tmpInSize = 4;
+ dctx->tmpInTarget = 8;
+ dctx->dStage = dstage_storeSFrameSize;
}
- if (dctxPtr->dStage == dstage_storeSFrameSize)
+ if (dctx->dStage == dstage_storeSFrameSize)
case dstage_storeSFrameSize:
{
- size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize;
- if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr;
- memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
+ size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize,
+ (size_t)(srcEnd - srcPtr) );
+ memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);
srcPtr += sizeToCopy;
- dctxPtr->tmpInSize += sizeToCopy;
- if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) { /* not enough input to get full sBlockSize; wait for more */
- nextSrcSizeHint = dctxPtr->tmpInTarget - dctxPtr->tmpInSize;
+ dctx->tmpInSize += sizeToCopy;
+ if (dctx->tmpInSize < dctx->tmpInTarget) {
+ /* not enough input to get full sBlockSize; wait for more */
+ nextSrcSizeHint = dctx->tmpInTarget - dctx->tmpInSize;
doAnotherStage = 0;
break;
}
- selectedIn = dctxPtr->header + 4;
+ selectedIn = dctx->header + 4;
}
/* case dstage_decodeSFrameSize: */ /* no direct access */
{ size_t const SFrameSize = LZ4F_readLE32(selectedIn);
- dctxPtr->frameInfo.contentSize = SFrameSize;
- dctxPtr->tmpInTarget = SFrameSize;
- dctxPtr->dStage = dstage_skipSkippable;
+ dctx->frameInfo.contentSize = SFrameSize;
+ dctx->tmpInTarget = SFrameSize;
+ dctx->dStage = dstage_skipSkippable;
break;
}
case dstage_skipSkippable:
- { size_t skipSize = dctxPtr->tmpInTarget;
- if (skipSize > (size_t)(srcEnd-srcPtr)) skipSize = srcEnd-srcPtr;
+ { size_t const skipSize = MIN(dctx->tmpInTarget, (size_t)(srcEnd-srcPtr));
srcPtr += skipSize;
- dctxPtr->tmpInTarget -= skipSize;
+ dctx->tmpInTarget -= skipSize;
doAnotherStage = 0;
- nextSrcSizeHint = dctxPtr->tmpInTarget;
- if (nextSrcSizeHint) break;
- dctxPtr->dStage = dstage_getHeader;
+ nextSrcSizeHint = dctx->tmpInTarget;
+ if (nextSrcSizeHint) break; /* still more to skip */
+ LZ4F_resetDecompressionContext(dctx);
break;
}
}
}
- /* preserve dictionary within tmp if necessary */
- if ( (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked)
- &&(dctxPtr->dict != dctxPtr->tmpOutBuffer)
- &&(!decompressOptionsPtr->stableDst)
- &&((unsigned)(dctxPtr->dStage-1) < (unsigned)(dstage_getSuffix-1))
- )
+ /* preserve history within tmp if necessary */
+ if ( (dctx->frameInfo.blockMode==LZ4F_blockLinked)
+ && (dctx->dict != dctx->tmpOutBuffer)
+ && (dctx->dStage != dstage_getFrameHeader)
+ && (!decompressOptionsPtr->stableDst)
+ && ((unsigned)(dctx->dStage-1) < (unsigned)(dstage_getSuffix-1)) )
{
- if (dctxPtr->dStage == dstage_flushOut) {
- size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer;
- size_t copySize = 64 KB - dctxPtr->tmpOutSize;
- const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart;
- if (dctxPtr->tmpOutSize > 64 KB) copySize = 0;
+ if (dctx->dStage == dstage_flushOut) {
+ size_t preserveSize = dctx->tmpOut - dctx->tmpOutBuffer;
+ size_t copySize = 64 KB - dctx->tmpOutSize;
+ const BYTE* oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart;
+ if (dctx->tmpOutSize > 64 KB) copySize = 0;
if (copySize > preserveSize) copySize = preserveSize;
- memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize);
+ memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize);
- dctxPtr->dict = dctxPtr->tmpOutBuffer;
- dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart;
+ dctx->dict = dctx->tmpOutBuffer;
+ dctx->dictSize = preserveSize + dctx->tmpOutStart;
} else {
- size_t newDictSize = dctxPtr->dictSize;
- const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize;
+ size_t newDictSize = dctx->dictSize;
+ const BYTE* oldDictEnd = dctx->dict + dctx->dictSize;
if ((newDictSize) > 64 KB) newDictSize = 64 KB;
- memcpy(dctxPtr->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize);
+ memcpy(dctx->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize);
- dctxPtr->dict = dctxPtr->tmpOutBuffer;
- dctxPtr->dictSize = newDictSize;
- dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + newDictSize;
+ dctx->dict = dctx->tmpOutBuffer;
+ dctx->dictSize = newDictSize;
+ dctx->tmpOut = dctx->tmpOutBuffer + newDictSize;
}
}
@@ -1411,3 +1647,23 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
*dstSizePtr = (dstPtr - dstStart);
return nextSrcSizeHint;
}
+
+/*! LZ4F_decompress_usingDict() :
+ * Same as LZ4F_decompress(), using a predefined dictionary.
+ * Dictionary is used "in place", without any preprocessing.
+ * It must remain accessible throughout the entire frame decoding.
+ */
+size_t LZ4F_decompress_usingDict(LZ4F_dctx* dctx,
+ void* dstBuffer, size_t* dstSizePtr,
+ const void* srcBuffer, size_t* srcSizePtr,
+ const void* dict, size_t dictSize,
+ const LZ4F_decompressOptions_t* decompressOptionsPtr)
+{
+ if (dctx->dStage <= dstage_init) {
+ dctx->dict = (const BYTE*)dict;
+ dctx->dictSize = dictSize;
+ }
+ return LZ4F_decompress(dctx, dstBuffer, dstSizePtr,
+ srcBuffer, srcSizePtr,
+ decompressOptionsPtr);
+}
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index a4a4ce6..88a6513 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -1,7 +1,7 @@
/*
LZ4 auto-framing library
Header File
- Copyright (C) 2011-2016, Yann Collet.
+ Copyright (C) 2011-2017, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
@@ -48,49 +48,68 @@ extern "C" {
/* --- Dependency --- */
#include <stddef.h> /* size_t */
-/*-***************************************************************
-* Compiler specifics
-*****************************************************************/
-/*!
-* LZ4_DLL_EXPORT :
-* Enable exporting of functions when building a Windows DLL
+
+/**
+ Introduction
+
+ lz4frame.h implements LZ4 frame specification (doc/lz4_Frame_format.md).
+ lz4frame.h provides frame compression functions that take care
+ of encoding standard metadata alongside LZ4-compressed blocks.
*/
+
+/*-***************************************************************
+ * Compiler specifics
+ *****************************************************************/
+/* LZ4_DLL_EXPORT :
+ * Enable exporting of functions when building a Windows DLL
+ * LZ4FLIB_API :
+ * Control library symbols visibility.
+ */
#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1)
# define LZ4FLIB_API __declspec(dllexport)
#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1)
# define LZ4FLIB_API __declspec(dllimport)
+#elif defined(__GNUC__) && (__GNUC__ >= 4)
+# define LZ4FLIB_API __attribute__ ((__visibility__ ("default")))
#else
# define LZ4FLIB_API
#endif
-#if defined(_MSC_VER)
-# define LZ4F_DEPRECATE(x) x /* __declspec(deprecated) x - only works with C++ */
-#elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 6))
-# define LZ4F_DEPRECATE(x) x __attribute__((deprecated))
+#ifdef LZ4F_DISABLE_DEPRECATE_WARNINGS
+# define LZ4F_DEPRECATE(x) x
#else
-# define LZ4F_DEPRECATE(x) x /* no deprecation warning for this compiler */
+# if defined(_MSC_VER)
+# define LZ4F_DEPRECATE(x) x /* __declspec(deprecated) x - only works with C++ */
+# elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 6))
+# define LZ4F_DEPRECATE(x) x __attribute__((deprecated))
+# else
+# define LZ4F_DEPRECATE(x) x /* no deprecation warning for this compiler */
+# endif
#endif
/*-************************************
-* Error management
-**************************************/
+ * Error management
+ **************************************/
typedef size_t LZ4F_errorCode_t;
-LZ4FLIB_API unsigned LZ4F_isError(LZ4F_errorCode_t code);
-LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /* return error code string; useful for debugging */
+LZ4FLIB_API unsigned LZ4F_isError(LZ4F_errorCode_t code); /**< tells if a `LZ4F_errorCode_t` function result is an error code */
+LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return error code string; useful for debugging */
/*-************************************
-* Frame compression types
-**************************************/
-/* #define LZ4F_DISABLE_OBSOLETE_ENUMS */ /* uncomment to disable obsolete enums */
-#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS
+ * Frame compression types
+ **************************************/
+/* #define LZ4F_ENABLE_OBSOLETE_ENUMS // uncomment to enable obsolete enums */
+#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS
# define LZ4F_OBSOLETE_ENUM(x) , LZ4F_DEPRECATE(x) = LZ4F_##x
#else
# define LZ4F_OBSOLETE_ENUM(x)
#endif
+/* The larger the block size, the (slightly) better the compression ratio,
+ * though there are diminishing returns.
+ * Larger blocks also increase memory usage on both compression and decompression sides. */
typedef enum {
LZ4F_default=0,
LZ4F_max64KB=4,
@@ -103,6 +122,9 @@ typedef enum {
LZ4F_OBSOLETE_ENUM(max4MB)
} LZ4F_blockSizeID_t;
+/* Linked blocks sharply reduce inefficiencies when using small blocks,
+ * they compress better.
+ * However, some LZ4 decoders are only compatible with independent blocks */
typedef enum {
LZ4F_blockLinked=0,
LZ4F_blockIndependent
@@ -118,63 +140,70 @@ typedef enum {
} LZ4F_contentChecksum_t;
typedef enum {
+ LZ4F_noBlockChecksum=0,
+ LZ4F_blockChecksumEnabled
+} LZ4F_blockChecksum_t;
+
+typedef enum {
LZ4F_frame=0,
LZ4F_skippableFrame
LZ4F_OBSOLETE_ENUM(skippableFrame)
} LZ4F_frameType_t;
-#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS
+#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS
typedef LZ4F_blockSizeID_t blockSizeID_t;
typedef LZ4F_blockMode_t blockMode_t;
typedef LZ4F_frameType_t frameType_t;
typedef LZ4F_contentChecksum_t contentChecksum_t;
#endif
-/* LZ4F_frameInfo_t :
- * makes it possible to supply detailed frame parameters to the stream interface.
- * It's not required to set all fields, as long as the structure was initially memset() to zero.
- * All reserved fields must be set to zero. */
+/*! LZ4F_frameInfo_t :
+ * makes it possible to set or read frame parameters.
+ * It's not required to set all fields, as long as the structure was initially memset() to zero.
+ * For all fields, 0 sets it to default value */
typedef struct {
- LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
- LZ4F_blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */
- LZ4F_contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */
- LZ4F_frameType_t frameType; /* LZ4F_frame, skippableFrame ; 0 == default */
- unsigned long long contentSize; /* Size of uncompressed (original) content ; 0 == unknown */
- unsigned reserved[2]; /* must be zero for forward compatibility */
+ LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
+ LZ4F_blockMode_t blockMode; /* LZ4F_blockLinked, LZ4F_blockIndependent ; 0 == default */
+ LZ4F_contentChecksum_t contentChecksumFlag; /* if enabled, frame is terminated with a 32-bits checksum of decompressed data ; 0 == disabled (default) */
+ LZ4F_frameType_t frameType; /* read-only field : LZ4F_frame or LZ4F_skippableFrame */
+ unsigned long long contentSize; /* Size of uncompressed content ; 0 == unknown */
+ unsigned dictID; /* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */
+ LZ4F_blockChecksum_t blockChecksumFlag; /* if enabled, each block is followed by a checksum of block's compressed data ; 0 == disabled (default) */
} LZ4F_frameInfo_t;
-/* LZ4F_preferences_t :
- * makes it possible to supply detailed compression parameters to the stream interface.
- * It's not required to set all fields, as long as the structure was initially memset() to zero.
- * All reserved fields must be set to zero. */
+/*! LZ4F_preferences_t :
+ * makes it possible to supply detailed compression parameters to the stream interface.
+ * It's not required to set all fields, as long as the structure was initially memset() to zero.
+ * All reserved fields must be set to zero. */
typedef struct {
LZ4F_frameInfo_t frameInfo;
- int compressionLevel; /* 0 == default (fast mode); values above 16 count as 16; values below 0 count as 0 */
- unsigned autoFlush; /* 1 == always flush (reduce usage of tmp buffer) */
+ int compressionLevel; /* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */
+ unsigned autoFlush; /* 1 == always flush, to reduce usage of internal buffers */
unsigned reserved[4]; /* must be zero for forward compatibility */
} LZ4F_preferences_t;
+LZ4FLIB_API int LZ4F_compressionLevel_max(void);
+
/*-*********************************
* Simple compression function
***********************************/
-/*!LZ4F_compressFrameBound() :
- * Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
- * Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
+/*! LZ4F_compressFrameBound() :
+ * Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
+ * Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
*/
LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
-/*!LZ4F_compressFrame() :
- * Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.1
- * An important rule is that dstBuffer MUST be large enough (dstCapacity) to store the result in worst case situation.
- * This value is supplied by LZ4F_compressFrameBound().
- * If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode).
- * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
+/*! LZ4F_compressFrame() :
+ * Compress an entire srcBuffer into a valid LZ4 frame.
+ * dstCapacity MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
+ * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
* @return : number of bytes written into dstBuffer.
* or an error code if it fails (can be tested using LZ4F_isError())
*/
-LZ4FLIB_API size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
-
+LZ4FLIB_API size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
+ const void* srcBuffer, size_t srcSize,
+ const LZ4F_preferences_t* preferencesPtr);
/*-***********************************
@@ -188,44 +217,45 @@ typedef struct {
unsigned reserved[3];
} LZ4F_compressOptions_t;
-/* Resource Management */
+/*--- Resource Management ---*/
#define LZ4F_VERSION 100
LZ4FLIB_API unsigned LZ4F_getVersion(void);
-LZ4FLIB_API LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version);
-LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
-/* LZ4F_createCompressionContext() :
+/*! LZ4F_createCompressionContext() :
* The first thing to do is to create a compressionContext object, which will be used in all compression operations.
- * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure.
+ * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version.
* The version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL.
* The function will provide a pointer to a fully allocated LZ4F_cctx object.
* If @return != zero, there was an error during context creation.
* Object can release its memory using LZ4F_freeCompressionContext();
*/
+LZ4FLIB_API LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version);
+LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
-/* Compression */
+/*---- Compression ----*/
-#define LZ4F_HEADER_SIZE_MAX 15
-LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* prefsPtr);
-/* LZ4F_compressBegin() :
- * will write the frame header into dstBuffer.
- * dstCapacity must be large enough to store the header. Maximum header size is LZ4F_HEADER_SIZE_MAX bytes.
+#define LZ4F_HEADER_SIZE_MAX 19
+/*! LZ4F_compressBegin() :
+ * will write the frame header into dstBuffer.
+ * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
* `prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default.
* @return : number of bytes written into dstBuffer for the header
* or an error code (which can be tested using LZ4F_isError())
*/
+LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx,
+ void* dstBuffer, size_t dstCapacity,
+ const LZ4F_preferences_t* prefsPtr);
-LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr);
-/* LZ4F_compressBound() :
+/*! LZ4F_compressBound() :
* Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
* prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario.
* Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
* When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
*/
+LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr);
-LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr);
-/* LZ4F_compressUpdate() :
+/*! LZ4F_compressUpdate() :
* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
* An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations.
* This value is provided by LZ4F_compressBound().
@@ -235,9 +265,9 @@ LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t
* @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
* or an error code if it fails (which can be tested using LZ4F_isError())
*/
+LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr);
-LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
-/* LZ4F_flush() :
+/*! LZ4F_flush() :
* When data must be generated and sent immediately, without waiting for a block to be completely filled,
* it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.
* `dstCapacity` must be large enough to ensure the operation will be successful.
@@ -245,17 +275,18 @@ LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapaci
* @return : number of bytes written into dstBuffer (it can be zero, which means there was no data stored within cctx)
* or an error code if it fails (which can be tested using LZ4F_isError())
*/
+LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
-LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
-/* LZ4F_compressEnd() :
- * To properly finish an LZ4 frame, invoke LZ4F_compressEnd().
- * It will flush whatever data remained within `cctx` (like LZ4_flush())
- * and properly finalize the frame, with an endMark and a checksum.
+/*! LZ4F_compressEnd() :
+ * To properly finish an LZ4 frame, invoke LZ4F_compressEnd().
+ * It will flush whatever data remained within `cctx` (like LZ4_flush())
+ * and properly finalize the frame, with an endMark and a checksum.
* `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default.
* @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled)
* or an error code if it fails (which can be tested using LZ4F_isError())
- * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.
+ * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.
*/
+LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
/*-*********************************
@@ -265,66 +296,78 @@ typedef struct LZ4F_dctx_s LZ4F_dctx; /* incomplete type */
typedef LZ4F_dctx* LZ4F_decompressionContext_t; /* compatibility with previous API versions */
typedef struct {
- unsigned stableDst; /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */
- unsigned reserved[3];
+ unsigned stableDst; /* pledge that at least 64KB+64Bytes of previously decompressed data remain unmodifed where it was decoded. This optimization skips storage operations in tmp buffers */
+ unsigned reserved[3]; /* must be set to zero for forward compatibility */
} LZ4F_decompressOptions_t;
/* Resource management */
/*!LZ4F_createDecompressionContext() :
- * Create an LZ4F_decompressionContext_t object, which will be used to track all decompression operations.
- * The version provided MUST be LZ4F_VERSION. It is intended to track potential breaking differences between different versions.
- * The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext_t object.
+ * Create an LZ4F_dctx object, to track all decompression operations.
+ * The version provided MUST be LZ4F_VERSION.
+ * The function provides a pointer to an allocated and initialized LZ4F_dctx object.
* The result is an errorCode, which can be tested using LZ4F_isError().
* dctx memory can be released using LZ4F_freeDecompressionContext();
* The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
* That is, it should be == 0 if decompression has been completed fully and correctly.
*/
LZ4FLIB_API LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version);
-LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctx);
+LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
-/*====== Decompression ======*/
+/*-***********************************
+* Streaming decompression functions
+*************************************/
-/*!LZ4F_getFrameInfo() :
- * This function decodes frame header information (such as max blockSize, frame checksum, etc.).
- * Its usage is optional. The objective is to extract frame header information, typically for allocation purposes.
- * A header size is variable and can length from 7 to 15 bytes. It's possible to provide more input bytes than that.
- * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
- * Decompression must resume from this point (srcBuffer + *srcSizePtr).
- * Note that LZ4F_getFrameInfo() can also be used anytime *after* decompression is started, in which case 0 input byte can be enough.
- * Frame header info is *copied into* an already allocated LZ4F_frameInfo_t structure.
+/*! LZ4F_getFrameInfo() :
+ * This function extracts frame parameters (max blockSize, dictID, etc.).
+ * Its usage is optional.
+ * Extracted information is typically useful for allocation and dictionary.
+ * This function works in 2 situations :
+ * - At the beginning of a new frame, in which case
+ * it will decode information from `srcBuffer`, starting the decoding process.
+ * Input size must be large enough to successfully decode the entire frame header.
+ * Frame header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes.
+ * It's allowed to provide more input data than this minimum.
+ * - After decoding has been started.
+ * In which case, no input is read, frame parameters are extracted from dctx.
+ * - If decoding has barely started, but not yet extracted information from header,
+ * LZ4F_getFrameInfo() will fail.
+ * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
+ * Decompression must resume from (srcBuffer + *srcSizePtr).
* @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
- * or an error code which can be tested using LZ4F_isError()
- * (typically, when there is not enough src bytes to fully decode the frame header)
+ * or an error code which can be tested using LZ4F_isError().
+ * note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely.
+ * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
*/
LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
LZ4F_frameInfo_t* frameInfoPtr,
const void* srcBuffer, size_t* srcSizePtr);
-/*!LZ4F_decompress() :
- * Call this function repetitively to regenerate data compressed within `srcBuffer`.
- * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
+/*! LZ4F_decompress() :
+ * Call this function repetitively to regenerate compressed data from `srcBuffer`.
+ * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
*
- * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
+ * The number of bytes regenerated into dstBuffer is provided within *dstSizePtr (necessarily <= original value).
*
- * The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
- * Number of bytes read can be < number of bytes provided, meaning there is some more data to decode.
- * It typically happens when dstBuffer is not large enough to contain all decoded data.
- * Remaining data will have to be presented again in a subsequent invocation.
+ * The number of bytes consumed from srcBuffer is provided within *srcSizePtr (necessarily <= original value).
+ * Number of bytes consumed can be < number of bytes provided.
+ * It typically happens when dstBuffer is not large enough to contain all decoded data.
+ * Unconsumed source data must be presented again in subsequent invocations.
*
* `dstBuffer` content is expected to be flushed between each invocation, as its content will be overwritten.
- * `dstBuffer` can be changed at will between each consecutive function invocation.
+ * `dstBuffer` itself can be changed at will between each consecutive function invocation.
*
- * @return is an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
- * Schematically, it's the size of the current (or remaining) compressed block + header of next block.
- * Respecting the hint provides some boost to performance, since it does skip intermediate buffers.
- * This is just a hint though, it's always possible to provide any srcSize.
- * When a frame is fully decoded, @return will be 0 (no more data expected).
- * If decompression failed, @return is an error code, which can be tested using LZ4F_isError().
+ * @return : an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
+ * Schematically, it's the size of the current (or remaining) compressed block + header of next block.
+ * Respecting the hint provides some small speed benefit, because it skips intermediate buffers.
+ * This is just a hint though, it's always possible to provide any srcSize.
+ * When a frame is fully decoded, @return will be 0 (no more data expected).
+ * If decompression failed, @return is an error code, which can be tested using LZ4F_isError().
*
- * After a frame is fully decoded, dctx can be used again to decompress another frame.
+ * After a frame is fully decoded, dctx can be used again to decompress another frame.
+ * After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state.
*/
LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx,
void* dstBuffer, size_t* dstSizePtr,
@@ -332,6 +375,14 @@ LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx,
const LZ4F_decompressOptions_t* dOptPtr);
+/*! LZ4F_resetDecompressionContext() : v1.8.0
+ * In case of an error, the context is left in "undefined" state.
+ * In which case, it's necessary to reset it, before re-using it.
+ * This method can also be used to abruptly stop an unfinished decompression,
+ * and start a new one using the same context. */
+LZ4FLIB_API void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx); /* always successful */
+
+
#if defined (__cplusplus)
}
diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h
index f2228a5..1899f8e 100644
--- a/lib/lz4frame_static.h
+++ b/lib/lz4frame_static.h
@@ -43,7 +43,7 @@ extern "C" {
/* lz4frame_static.h should be used solely in the context of static linking.
* It contains definitions which are not stable and may change in the future.
* Never use it in the context of DLL linking.
- * */
+ */
/* --- Dependency --- */
@@ -52,29 +52,90 @@ extern "C" {
/* --- Error List --- */
#define LZ4F_LIST_ERRORS(ITEM) \
- ITEM(OK_NoError) ITEM(ERROR_GENERIC) \
- ITEM(ERROR_maxBlockSize_invalid) ITEM(ERROR_blockMode_invalid) ITEM(ERROR_contentChecksumFlag_invalid) \
+ ITEM(OK_NoError) \
+ ITEM(ERROR_GENERIC) \
+ ITEM(ERROR_maxBlockSize_invalid) \
+ ITEM(ERROR_blockMode_invalid) \
+ ITEM(ERROR_contentChecksumFlag_invalid) \
ITEM(ERROR_compressionLevel_invalid) \
- ITEM(ERROR_headerVersion_wrong) ITEM(ERROR_blockChecksum_unsupported) ITEM(ERROR_reservedFlag_set) \
+ ITEM(ERROR_headerVersion_wrong) \
+ ITEM(ERROR_blockChecksum_invalid) \
+ ITEM(ERROR_reservedFlag_set) \
ITEM(ERROR_allocation_failed) \
- ITEM(ERROR_srcSize_tooLarge) ITEM(ERROR_dstMaxSize_tooSmall) \
- ITEM(ERROR_frameHeader_incomplete) ITEM(ERROR_frameType_unknown) ITEM(ERROR_frameSize_wrong) \
+ ITEM(ERROR_srcSize_tooLarge) \
+ ITEM(ERROR_dstMaxSize_tooSmall) \
+ ITEM(ERROR_frameHeader_incomplete) \
+ ITEM(ERROR_frameType_unknown) \
+ ITEM(ERROR_frameSize_wrong) \
ITEM(ERROR_srcPtr_wrong) \
ITEM(ERROR_decompressionFailed) \
- ITEM(ERROR_headerChecksum_invalid) ITEM(ERROR_contentChecksum_invalid) \
+ ITEM(ERROR_headerChecksum_invalid) \
+ ITEM(ERROR_contentChecksum_invalid) \
+ ITEM(ERROR_frameDecoding_alreadyStarted) \
ITEM(ERROR_maxCode)
-#define LZ4F_DISABLE_OLD_ENUMS /* comment to enable deprecated enums */
-#ifndef LZ4F_DISABLE_OLD_ENUMS
-# define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM, ENUM = LZ4F_##ENUM,
-#else
-# define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM,
-#endif
-typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; /* enum is exposed, to handle specific errors; compare function result to -enum value */
+#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM,
+
+/* enum list is exposed, to handle specific errors */
+typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes;
LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult);
+
+/**********************************
+ * Bulk processing dictionary API
+ *********************************/
+typedef struct LZ4F_CDict_s LZ4F_CDict;
+
+/*! LZ4_createCDict() :
+ * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
+ * LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+ * LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
+ * `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict */
+LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize);
+void LZ4F_freeCDict(LZ4F_CDict* CDict);
+
+
+/*! LZ4_compressFrame_usingCDict() :
+ * Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary.
+ * If cdict==NULL, compress without a dictionary.
+ * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
+ * If this condition is not respected, function will fail (@return an errorCode).
+ * The LZ4F_preferences_t structure is optional : you may provide NULL as argument,
+ * but it's not recommended, as it's the only way to provide dictID in the frame header.
+ * @return : number of bytes written into dstBuffer.
+ * or an error code if it fails (can be tested using LZ4F_isError()) */
+size_t LZ4F_compressFrame_usingCDict(void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const LZ4F_CDict* cdict,
+ const LZ4F_preferences_t* preferencesPtr);
+
+
+/*! LZ4F_compressBegin_usingCDict() :
+ * Inits streaming dictionary compression, and writes the frame header into dstBuffer.
+ * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
+ * `prefsPtr` is optional : you may provide NULL as argument,
+ * however, it's the only way to provide dictID in the frame header.
+ * @return : number of bytes written into dstBuffer for the header,
+ * or an error code (which can be tested using LZ4F_isError()) */
+size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctx,
+ void* dstBuffer, size_t dstCapacity,
+ const LZ4F_CDict* cdict,
+ const LZ4F_preferences_t* prefsPtr);
+
+
+/*! LZ4F_decompress_usingDict() :
+ * Same as LZ4F_decompress(), using a predefined dictionary.
+ * Dictionary is used "in place", without any preprocessing.
+ * It must remain accessible throughout the entire frame decoding. */
+size_t LZ4F_decompress_usingDict(LZ4F_dctx* dctxPtr,
+ void* dstBuffer, size_t* dstSizePtr,
+ const void* srcBuffer, size_t* srcSizePtr,
+ const void* dict, size_t dictSize,
+ const LZ4F_decompressOptions_t* decompressOptionsPtr);
+
+
#if defined (__cplusplus)
}
#endif
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 5d4ea3e..22eb071 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -1,6 +1,6 @@
/*
LZ4 HC - High Compression Mode of LZ4
- Copyright (C) 2011-2016, Yann Collet.
+ Copyright (C) 2011-2017, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@@ -38,54 +38,42 @@
* Tuning Parameter
***************************************/
-/*!
- * HEAPMODE :
- * Select how default compression function will allocate workplace memory,
- * in stack (0:fastest), or in heap (1:requires malloc()).
- * Since workplace is rather large, heap mode is recommended.
+/*! HEAPMODE :
+ * Select how default compression function will allocate workplace memory,
+ * in stack (0:fastest), or in heap (1:requires malloc()).
+ * Since workplace is rather large, heap mode is recommended.
*/
#ifndef LZ4HC_HEAPMODE
# define LZ4HC_HEAPMODE 1
#endif
-/* *************************************
-* Dependency
-***************************************/
+/*=== Dependency ===*/
#include "lz4hc.h"
-/* *************************************
-* Local Compiler Options
-***************************************/
+/*=== Common LZ4 definitions ===*/
#if defined(__GNUC__)
# pragma GCC diagnostic ignored "-Wunused-function"
#endif
-
#if defined (__clang__)
# pragma clang diagnostic ignored "-Wunused-function"
#endif
-
-/* *************************************
-* Common LZ4 definition
-***************************************/
#define LZ4_COMMONDEFS_ONLY
-#include "lz4.c"
+#include "lz4.c" /* LZ4_count, constants, mem */
-/* *************************************
-* Local Constants
-***************************************/
+/*=== Constants ===*/
#define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH)
-/**************************************
-* Local Macros
-**************************************/
-#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG))
-#define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */
-#define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */
+/*=== Macros ===*/
+#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
+#define MAX(a,b) ( (a) > (b) ? (a) : (b) )
+#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG))
+#define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */
+#define DELTANEXTU16(table, pos) table[(U16)(pos)] /* faster */
static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); }
@@ -120,7 +108,7 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
U32 const h = LZ4HC_hashPtr(base+idx);
size_t delta = idx - hashTable[h];
if (delta>MAX_DISTANCE) delta = MAX_DISTANCE;
- DELTANEXTU16(idx) = (U16)delta;
+ DELTANEXTU16(chainTable, idx) = (U16)delta;
hashTable[h] = idx;
idx++;
}
@@ -129,8 +117,8 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
}
-FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* hc4, /* Index table will be updated */
- const BYTE* ip, const BYTE* const iLimit,
+FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* const hc4, /* Index table will be updated */
+ const BYTE* const ip, const BYTE* const iLimit,
const BYTE** matchpos,
const int maxNbAttempts)
{
@@ -141,8 +129,8 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* hc4, /* In
const U32 dictLimit = hc4->dictLimit;
const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1);
U32 matchIndex;
- int nbAttempts=maxNbAttempts;
- size_t ml=0;
+ int nbAttempts = maxNbAttempts;
+ size_t ml = 0;
/* HC4 match finder */
LZ4HC_Insert(hc4, ip);
@@ -152,8 +140,8 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* hc4, /* In
nbAttempts--;
if (matchIndex >= dictLimit) {
const BYTE* const match = base + matchIndex;
- if (*(match+ml) == *(ip+ml)
- && (LZ4_read32(match) == LZ4_read32(ip)))
+ if ( (*(match+ml) == *(ip+ml)) /* can be longer */
+ && (LZ4_read32(match) == LZ4_read32(ip)) )
{
size_t const mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
if (mlt > ml) { ml = mlt; *matchpos = match; }
@@ -170,7 +158,7 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* hc4, /* In
if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */
}
}
- matchIndex -= DELTANEXTU16(matchIndex);
+ matchIndex -= DELTANEXTU16(chainTable, matchIndex);
}
return (int)ml;
@@ -194,9 +182,9 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
const BYTE* const lowPrefixPtr = base + dictLimit;
const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1);
const BYTE* const dictBase = hc4->dictBase;
- U32 matchIndex;
+ int const delta = (int)(ip-iLowLimit);
int nbAttempts = maxNbAttempts;
- int delta = (int)(ip-iLowLimit);
+ U32 matchIndex;
/* First Match */
@@ -206,30 +194,29 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
while ((matchIndex>=lowLimit) && (nbAttempts)) {
nbAttempts--;
if (matchIndex >= dictLimit) {
- const BYTE* matchPtr = base + matchIndex;
+ const BYTE* const matchPtr = base + matchIndex;
if (*(iLowLimit + longest) == *(matchPtr - delta + longest)) {
if (LZ4_read32(matchPtr) == LZ4_read32(ip)) {
int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
int back = 0;
- while ((ip+back > iLowLimit)
- && (matchPtr+back > lowPrefixPtr)
- && (ip[back-1] == matchPtr[back-1]))
+ while ( (ip+back > iLowLimit)
+ && (matchPtr+back > lowPrefixPtr)
+ && (ip[back-1] == matchPtr[back-1])) {
back--;
+ }
mlt -= back;
if (mlt > longest) {
- longest = (int)mlt;
+ longest = mlt;
*matchpos = matchPtr+back;
*startpos = ip+back;
- }
- }
- }
+ } } }
} else {
const BYTE* const matchPtr = dictBase + matchIndex;
if (LZ4_read32(matchPtr) == LZ4_read32(ip)) {
- size_t mlt;
+ int mlt;
int back=0;
const BYTE* vLimit = ip + (dictLimit - matchIndex);
if (vLimit > iHighLimit) vLimit = iHighLimit;
@@ -238,45 +225,57 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit);
while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchPtr[back-1])) back--;
mlt -= back;
- if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; }
+ if (mlt > longest) { longest = mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; }
}
}
- matchIndex -= DELTANEXTU16(matchIndex);
+ matchIndex -= DELTANEXTU16(chainTable, matchIndex);
}
return longest;
}
-typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive;
+typedef enum {
+ noLimit = 0,
+ limitedOutput = 1,
+ limitedDestSize = 2,
+} limitedOutput_directive;
-#define LZ4HC_DEBUG 0
-#if LZ4HC_DEBUG
-static unsigned debug = 0;
+#ifndef LZ4HC_DEBUG
+# define LZ4HC_DEBUG 0
#endif
+/* LZ4HC_encodeSequence() :
+ * @return : 0 if ok,
+ * 1 if buffer issue detected */
FORCE_INLINE int LZ4HC_encodeSequence (
const BYTE** ip,
BYTE** op,
const BYTE** anchor,
int matchLength,
const BYTE* const match,
- limitedOutput_directive limitedOutputBuffer,
+ limitedOutput_directive limit,
BYTE* oend)
{
- int length;
- BYTE* token;
+ size_t length;
+ BYTE* const token = (*op)++;
#if LZ4HC_DEBUG
- if (debug) printf("literal : %u -- match : %u -- offset : %u\n", (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match));
+ printf("literal : %u -- match : %u -- offset : %u\n",
+ (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match));
#endif
/* Encode Literal length */
- length = (int)(*ip - *anchor);
- token = (*op)++;
- if ((limitedOutputBuffer) && ((*op + (length>>8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */
- if (length>=(int)RUN_MASK) { int len; *token=(RUN_MASK<<ML_BITS); len = length-RUN_MASK; for(; len > 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; }
- else *token = (BYTE)(length<<ML_BITS);
+ length = (size_t)(*ip - *anchor);
+ if ((limit) && ((*op + (length >> 8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */
+ if (length >= RUN_MASK) {
+ size_t len = length - RUN_MASK;
+ *token = (RUN_MASK << ML_BITS);
+ for(; len >= 255 ; len -= 255) *(*op)++ = 255;
+ *(*op)++ = (BYTE)len;
+ } else {
+ *token = (BYTE)(length << ML_BITS);
+ }
/* Copy Literals */
LZ4_wildCopy(*op, *anchor, (*op) + length);
@@ -286,13 +285,13 @@ FORCE_INLINE int LZ4HC_encodeSequence (
LZ4_writeLE16(*op, (U16)(*ip-match)); *op += 2;
/* Encode MatchLength */
- length = (int)(matchLength-MINMATCH);
- if ((limitedOutputBuffer) && (*op + (length>>8) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */
- if (length>=(int)ML_MASK) {
+ length = (size_t)(matchLength - MINMATCH);
+ if ((limit) && (*op + (length >> 8) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */
+ if (length >= ML_MASK) {
*token += ML_MASK;
length -= ML_MASK;
- for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; }
- if (length > 254) { length-=255; *(*op)++ = 255; }
+ for(; length >= 510 ; length -= 510) { *(*op)++ = 255; *(*op)++ = 255; }
+ if (length >= 255) { length -= 255; *(*op)++ = 255; }
*(*op)++ = (BYTE)length;
} else {
*token += (BYTE)(length);
@@ -305,26 +304,31 @@ FORCE_INLINE int LZ4HC_encodeSequence (
return 0;
}
+/* btopt */
#include "lz4opt.h"
+
static int LZ4HC_compress_hashChain (
LZ4HC_CCtx_internal* const ctx,
const char* const source,
char* const dest,
- int const inputSize,
+ int* srcSizePtr,
int const maxOutputSize,
unsigned maxNbAttempts,
limitedOutput_directive limit
)
{
+ const int inputSize = *srcSizePtr;
+
const BYTE* ip = (const BYTE*) source;
const BYTE* anchor = ip;
const BYTE* const iend = ip + inputSize;
const BYTE* const mflimit = iend - MFLIMIT;
const BYTE* const matchlimit = (iend - LASTLITERALS);
+ BYTE* optr = (BYTE*) dest;
BYTE* op = (BYTE*) dest;
- BYTE* const oend = op + maxOutputSize;
+ BYTE* oend = op + maxOutputSize;
int ml, ml2, ml3, ml0;
const BYTE* ref = NULL;
@@ -336,7 +340,13 @@ static int LZ4HC_compress_hashChain (
const BYTE* ref0;
/* init */
+ *srcSizePtr = 0;
+ if (limit == limitedDestSize && maxOutputSize < 1) return 0; /* Impossible to store anything */
+ if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */
+
ctx->end += inputSize;
+ if (limit == limitedDestSize) oend -= LASTLITERALS; /* Hack for support limitations LZ4 decompressor */
+ if (inputSize < LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */
ip++;
@@ -353,10 +363,12 @@ static int LZ4HC_compress_hashChain (
_Search2:
if (ip+ml < mflimit)
ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2, maxNbAttempts);
- else ml2 = ml;
+ else
+ ml2 = ml;
if (ml2 == ml) { /* No better match */
- if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
+ optr = op;
+ if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow;
continue;
}
@@ -377,11 +389,9 @@ _Search2:
}
_Search3:
- /*
- * Currently we have :
- * ml2 > ml1, and
- * ip1+3 <= ip2 (usually < ip1+ml1)
- */
+ /* At this stage, we have :
+ * ml2 > ml1, and
+ * ip1+3 <= ip2 (usually < ip1+ml1) */
if ((start2 - ip) < OPTIMAL_ML) {
int correction;
int new_ml = ml;
@@ -398,15 +408,18 @@ _Search3:
if (start2 + ml2 < mflimit)
ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, maxNbAttempts);
- else ml3 = ml2;
+ else
+ ml3 = ml2;
if (ml3 == ml2) { /* No better match : 2 sequences to encode */
/* ip & ref are known; Now for ml */
if (start2 < ip+ml) ml = (int)(start2 - ip);
/* Now, encode 2 sequences */
- if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
+ optr = op;
+ if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow;
ip = start2;
- if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) return 0;
+ optr = op;
+ if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) goto _dest_overflow;
continue;
}
@@ -424,7 +437,8 @@ _Search3:
}
}
- if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
+ optr = op;
+ if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow;
ip = start3;
ref = ref3;
ml = ml3;
@@ -460,7 +474,8 @@ _Search3:
ml = (int)(start2 - ip);
}
}
- if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
+ optr = op;
+ if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow;
ip = start2;
ref = ref2;
@@ -473,65 +488,99 @@ _Search3:
goto _Search3;
}
+_last_literals:
/* Encode Last Literals */
- { int lastRun = (int)(iend - anchor);
- if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */
- if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun > 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
- else *op++ = (BYTE)(lastRun<<ML_BITS);
- memcpy(op, anchor, iend - anchor);
- op += iend-anchor;
+ { size_t lastRunSize = (size_t)(iend - anchor); /* literals */
+ size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255;
+ size_t const totalSize = 1 + litLength + lastRunSize;
+ if (limit == limitedDestSize) oend += LASTLITERALS; /* restore correct value */
+ if (limit && (op + totalSize > oend)) {
+ if (limit == limitedOutput) return 0; /* Check output limit */
+ /* adapt lastRunSize to fill 'dest' */
+ lastRunSize = (size_t)(oend - op) - 1;
+ litLength = (lastRunSize + 255 - RUN_MASK) / 255;
+ lastRunSize -= litLength;
+ }
+ ip = anchor + lastRunSize;
+
+ if (lastRunSize >= RUN_MASK) {
+ size_t accumulator = lastRunSize - RUN_MASK;
+ *op++ = (RUN_MASK << ML_BITS);
+ for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255;
+ *op++ = (BYTE) accumulator;
+ } else {
+ *op++ = (BYTE)(lastRunSize << ML_BITS);
+ }
+ memcpy(op, anchor, lastRunSize);
+ op += lastRunSize;
}
/* End */
+ *srcSizePtr = (int) (((const char*)ip) - source);
return (int) (((char*)op)-dest);
+
+_dest_overflow:
+ if (limit == limitedDestSize) {
+ op = optr; /* restore correct out pointer */
+ goto _last_literals;
+ }
+ return 0;
}
static int LZ4HC_getSearchNum(int compressionLevel)
{
switch (compressionLevel) {
default: return 0; /* unused */
- case 11: return 128;
- case 12: return 1<<10;
+ case 11: return 128;
+ case 12: return 1<<10;
}
}
static int LZ4HC_compress_generic (
LZ4HC_CCtx_internal* const ctx,
- const char* const source,
- char* const dest,
- int const inputSize,
- int const maxOutputSize,
- int compressionLevel,
+ const char* const src,
+ char* const dst,
+ int* const srcSizePtr,
+ int const dstCapacity,
+ int cLevel,
limitedOutput_directive limit
)
{
- if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT;
- if (compressionLevel > 9) {
- switch (compressionLevel) {
- case 10: return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (16-1), limit);
- case 11: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 128, 0);
+ if (cLevel < 1) cLevel = LZ4HC_CLEVEL_DEFAULT; /* note : convention is different from lz4frame, maybe to reconsider */
+ if (cLevel > 9) {
+ if (limit == limitedDestSize) cLevel = 10;
+ switch (cLevel) {
+ case 10:
+ return LZ4HC_compress_hashChain(ctx, src, dst, srcSizePtr, dstCapacity, 1 << 12, limit);
+ case 11:
+ ctx->searchNum = LZ4HC_getSearchNum(cLevel);
+ return LZ4HC_compress_optimal(ctx, src, dst, *srcSizePtr, dstCapacity, limit, 128, 0);
default:
- case 12: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM, 1);
+ cLevel = 12;
+ /* fall-through */
+ case 12:
+ ctx->searchNum = LZ4HC_getSearchNum(cLevel);
+ return LZ4HC_compress_optimal(ctx, src, dst, *srcSizePtr, dstCapacity, limit, LZ4_OPT_NUM, 1);
}
}
- return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (compressionLevel-1), limit);
+ return LZ4HC_compress_hashChain(ctx, src, dst, srcSizePtr, dstCapacity, 1 << (cLevel-1), limit); /* levels 1-9 */
}
int LZ4_sizeofStateHC(void) { return sizeof(LZ4_streamHC_t); }
-int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel)
+int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
{
- LZ4HC_CCtx_internal* ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
+ LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */
LZ4HC_init (ctx, (const BYTE*)src);
- if (maxDstSize < LZ4_compressBound(srcSize))
- return LZ4HC_compress_generic (ctx, src, dst, srcSize, maxDstSize, compressionLevel, limitedOutput);
+ if (dstCapacity < LZ4_compressBound(srcSize))
+ return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, limitedOutput);
else
- return LZ4HC_compress_generic (ctx, src, dst, srcSize, maxDstSize, compressionLevel, noLimit);
+ return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, noLimit);
}
-int LZ4_compress_HC(const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel)
+int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
{
#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
@@ -539,13 +588,23 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int maxDstSize, int
LZ4_streamHC_t state;
LZ4_streamHC_t* const statePtr = &state;
#endif
- int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, maxDstSize, compressionLevel);
+ int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
free(statePtr);
#endif
return cSize;
}
+/* LZ4_compress_HC_destSize() :
+ * currently, only compatible with Hash Chain implementation,
+ * hence limit compression level to LZ4HC_CLEVEL_OPT_MIN-1*/
+int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int cLevel)
+{
+ LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse;
+ LZ4HC_init(ctx, (const BYTE*) source);
+ return LZ4HC_compress_generic(ctx, source, dest, sourceSizePtr, targetDestSize, cLevel, limitedDestSize);
+}
+
/**************************************
@@ -553,7 +612,11 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int maxDstSize, int
**************************************/
/* allocation */
LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); }
-int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { free(LZ4_streamHCPtr); return 0; }
+int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) {
+ if (!LZ4_streamHCPtr) return 0; /* support free on NULL */
+ free(LZ4_streamHCPtr);
+ return 0;
+}
/* initialization */
@@ -561,13 +624,24 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
{
LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
LZ4_streamHCPtr->internal_donotuse.base = NULL;
- LZ4_streamHCPtr->internal_donotuse.compressionLevel = (unsigned)compressionLevel;
+ if (compressionLevel > LZ4HC_CLEVEL_MAX) compressionLevel = LZ4HC_CLEVEL_MAX; /* cap compression level */
+ LZ4_streamHCPtr->internal_donotuse.compressionLevel = compressionLevel;
LZ4_streamHCPtr->internal_donotuse.searchNum = LZ4HC_getSearchNum(compressionLevel);
}
+void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
+{
+ int const currentCLevel = LZ4_streamHCPtr->internal_donotuse.compressionLevel;
+ int const minCLevel = currentCLevel < LZ4HC_CLEVEL_OPT_MIN ? 1 : LZ4HC_CLEVEL_OPT_MIN;
+ int const maxCLevel = currentCLevel < LZ4HC_CLEVEL_OPT_MIN ? LZ4HC_CLEVEL_OPT_MIN-1 : LZ4HC_CLEVEL_MAX;
+ compressionLevel = MIN(compressionLevel, minCLevel);
+ compressionLevel = MAX(compressionLevel, maxCLevel);
+ LZ4_streamHCPtr->internal_donotuse.compressionLevel = compressionLevel;
+}
+
int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize)
{
- LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
+ LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
if (dictSize > 64 KB) {
dictionary += dictSize - 64 KB;
dictSize = 64 KB;
@@ -601,12 +675,13 @@ static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBl
}
static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr,
- const char* source, char* dest,
- int inputSize, int maxOutputSize, limitedOutput_directive limit)
+ const char* src, char* dst,
+ int* srcSizePtr, int dstCapacity,
+ limitedOutput_directive limit)
{
- LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
+ LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
/* auto-init if forgotten */
- if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) source);
+ if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) src);
/* Check overflow */
if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) {
@@ -616,31 +691,39 @@ static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr,
}
/* Check if blocks follow each other */
- if ((const BYTE*)source != ctxPtr->end) LZ4HC_setExternalDict(ctxPtr, (const BYTE*)source);
+ if ((const BYTE*)src != ctxPtr->end) LZ4HC_setExternalDict(ctxPtr, (const BYTE*)src);
/* Check overlapping input/dictionary space */
- { const BYTE* sourceEnd = (const BYTE*) source + inputSize;
+ { const BYTE* sourceEnd = (const BYTE*) src + *srcSizePtr;
const BYTE* const dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit;
const BYTE* const dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit;
- if ((sourceEnd > dictBegin) && ((const BYTE*)source < dictEnd)) {
+ if ((sourceEnd > dictBegin) && ((const BYTE*)src < dictEnd)) {
if (sourceEnd > dictEnd) sourceEnd = dictEnd;
ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase);
if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit;
}
}
- return LZ4HC_compress_generic (ctxPtr, source, dest, inputSize, maxOutputSize, ctxPtr->compressionLevel, limit);
+ return LZ4HC_compress_generic (ctxPtr, src, dst, srcSizePtr, dstCapacity, ctxPtr->compressionLevel, limit);
}
-int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize)
+int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int srcSize, int dstCapacity)
{
- if (maxOutputSize < LZ4_compressBound(inputSize))
- return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput);
+ if (dstCapacity < LZ4_compressBound(srcSize))
+ return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, limitedOutput);
else
- return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, noLimit);
+ return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, noLimit);
+}
+
+int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int* srcSizePtr, int targetDestSize)
+{
+ LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
+ if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN) LZ4HC_init(ctxPtr, (const BYTE*)src); /* not compatible with btopt implementation */
+ return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, dst, srcSizePtr, targetDestSize, limitedDestSize);
}
+
/* dictionary saving */
int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize)
@@ -700,16 +783,20 @@ void* LZ4_createHC (char* inputBuffer)
return hc4;
}
-int LZ4_freeHC (void* LZ4HC_Data) { FREEMEM(LZ4HC_Data); return 0; }
+int LZ4_freeHC (void* LZ4HC_Data) {
+ if (!LZ4HC_Data) return 0; /* support free on NULL */
+ FREEMEM(LZ4HC_Data);
+ return 0;
+}
-int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel)
+int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int cLevel)
{
- return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, source, dest, inputSize, 0, compressionLevel, noLimit);
+ return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, 0, cLevel, noLimit);
}
-int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
+int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int dstCapacity, int cLevel)
{
- return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
+ return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, dstCapacity, cLevel, limitedOutput);
}
char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 1036fd0..66d5636 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -1,7 +1,7 @@
/*
LZ4 HC - High Compression Mode of LZ4
Header File
- Copyright (C) 2011-2016, Yann Collet.
+ Copyright (C) 2011-2017, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
@@ -58,8 +58,8 @@ extern "C" {
* `dst` must be already allocated.
* Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h")
* Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h")
- * `compressionLevel` : Recommended values are between 4 and 9, although any value between 1 and LZ4HC_MAX_CLEVEL will work.
- * Values >LZ4HC_MAX_CLEVEL behave the same as LZ4HC_MAX_CLEVEL.
+ * `compressionLevel` : Recommended values are between 4 and 9, although any value between 1 and LZ4HC_CLEVEL_MAX will work.
+ * Values >LZ4HC_CLEVEL_MAX behave the same as LZ4HC_CLEVEL_MAX.
* @return : the number of bytes written into 'dst'
* or 0 if compression fails.
*/
@@ -113,25 +113,23 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in
Then, use LZ4_compress_HC_continue() to compress each successive block.
Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression.
- 'dst' buffer should be sized to handle worst case scenarios, using LZ4_compressBound(), to ensure operation success.
+ 'dst' buffer should be sized to handle worst case scenarios (see LZ4_compressBound()), to ensure operation success.
+ Because in case of failure, the API does not guarantee context recovery, and context will have to be reset.
+ If `dst` buffer budget cannot be >= LZ4_compressBound(), consider using LZ4_compress_HC_continue_destSize() instead.
- If, for any reason, previous data blocks can't be preserved unmodified in memory during next compression block,
- you must save it to a safer memory space, using LZ4_saveDictHC().
+ If, for any reason, previous data block can't be preserved unmodified in memory for next compression block,
+ you can save it to a more stable memory space, using LZ4_saveDictHC().
Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer'.
*/
-/*-******************************************
- * !!!!! STATIC LINKING ONLY !!!!!
- *******************************************/
-
/*-*************************************
* PRIVATE DEFINITIONS :
* Do not use these definitions.
* They are exposed to allow static allocation of `LZ4_streamHC_t`.
* Using these definitions makes the code vulnerable to potential API break when upgrading LZ4
**************************************/
-#define LZ4HC_DICTIONARY_LOGSIZE 17
+#define LZ4HC_DICTIONARY_LOGSIZE 17 /* because of btopt, hc would only need 16 */
#define LZ4HC_MAXD (1<<LZ4HC_DICTIONARY_LOGSIZE)
#define LZ4HC_MAXD_MASK (LZ4HC_MAXD - 1)
@@ -147,14 +145,14 @@ typedef struct
{
uint32_t hashTable[LZ4HC_HASHTABLESIZE];
uint16_t chainTable[LZ4HC_MAXD];
- const uint8_t* end; /* next block here to continue on current prefix */
- const uint8_t* base; /* All index relative to this position */
- const uint8_t* dictBase; /* alternate base for extDict */
- uint8_t* inputBuffer; /* deprecated */
- uint32_t dictLimit; /* below that point, need extDict */
- uint32_t lowLimit; /* below that point, no more dict */
- uint32_t nextToUpdate; /* index from which to continue dictionary update */
- uint32_t searchNum; /* only for optimal parser */
+ const uint8_t* end; /* next block here to continue on current prefix */
+ const uint8_t* base; /* All index relative to this position */
+ const uint8_t* dictBase; /* alternate base for extDict */
+ uint8_t* inputBuffer; /* deprecated */
+ uint32_t dictLimit; /* below that point, need extDict */
+ uint32_t lowLimit; /* below that point, no more dict */
+ uint32_t nextToUpdate; /* index from which to continue dictionary update */
+ uint32_t searchNum; /* only for optimal parser */
uint32_t compressionLevel;
} LZ4HC_CCtx_internal;
@@ -163,7 +161,7 @@ typedef struct
typedef struct
{
unsigned int hashTable[LZ4HC_HASHTABLESIZE];
- unsigned short chainTable[LZ4HC_MAXD];
+ unsigned short chainTable[LZ4HC_MAXD];
const unsigned char* end; /* next block here to continue on current prefix */
const unsigned char* base; /* All index relative to this position */
const unsigned char* dictBase; /* alternate base for extDict */
@@ -172,7 +170,7 @@ typedef struct
unsigned int lowLimit; /* below that point, no more dict */
unsigned int nextToUpdate; /* index from which to continue dictionary update */
unsigned int searchNum; /* only for optimal parser */
- unsigned int compressionLevel;
+ int compressionLevel;
} LZ4HC_CCtx_internal;
#endif
@@ -200,25 +198,25 @@ union LZ4_streamHC_u {
/* deprecated compression functions */
/* these functions will trigger warning messages in future releases */
-LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC (const char* source, char* dest, int inputSize);
-LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
-LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel);
-LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
-LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize);
-LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
-LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel);
-LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
-LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
-LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC (const char* source, char* dest, int inputSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
/* Deprecated Streaming functions using older model; should no longer be used */
-LZ4_DEPRECATED("use LZ4_createStreamHC() instead") void* LZ4_createHC (char* inputBuffer);
-LZ4_DEPRECATED("use LZ4_saveDictHC() instead") char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
-LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") int LZ4_freeHC (void* LZ4HC_Data);
-LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
-LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
-LZ4_DEPRECATED("use LZ4_createStreamHC() instead") int LZ4_sizeofStreamStateHC(void);
-LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStreamStateHC(void* state, char* inputBuffer);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStreamHC() instead") void* LZ4_createHC (char* inputBuffer);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_saveDictHC() instead") char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") int LZ4_freeHC (void* LZ4HC_Data);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStreamHC() instead") int LZ4_sizeofStreamStateHC(void);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStreamStateHC(void* state, char* inputBuffer);
#if defined (__cplusplus)
@@ -226,3 +224,55 @@ LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStreamStateHC(
#endif
#endif /* LZ4_HC_H_19834876238432 */
+
+/*-************************************************
+ * !!!!! STATIC LINKING ONLY !!!!!
+ * Following definitions are considered experimental.
+ * They should not be linked from DLL,
+ * as there is no guarantee of API stability yet.
+ * Prototypes will be promoted to "stable" status
+ * after successfull usage in real-life scenarios.
+ *************************************************/
+#ifdef LZ4_HC_STATIC_LINKING_ONLY /* protection macro */
+#ifndef LZ4_HC_SLO_098092834
+#define LZ4_HC_SLO_098092834
+
+/*! LZ4_compress_HC_destSize() : v1.8.0 (experimental)
+ * Will try to compress as much data from `src` as possible
+ * that can fit into `targetDstSize` budget.
+ * Result is provided in 2 parts :
+ * @return : the number of bytes written into 'dst'
+ * or 0 if compression fails.
+ * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src`
+ */
+int LZ4_compress_HC_destSize(void* LZ4HC_Data,
+ const char* src, char* dst,
+ int* srcSizePtr, int targetDstSize,
+ int compressionLevel);
+
+/*! LZ4_compress_HC_continue_destSize() : v1.8.0 (experimental)
+ * Similar as LZ4_compress_HC_continue(),
+ * but will read a variable nb of bytes from `src`
+ * to fit into `targetDstSize` budget.
+ * Result is provided in 2 parts :
+ * @return : the number of bytes written into 'dst'
+ * or 0 if compression fails.
+ * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src`.
+ * Important : due to limitations, this prototype only works well up to cLevel < LZ4HC_CLEVEL_OPT_MIN
+ * beyond that level, compression performance will be much reduced due to internal incompatibilities
+ */
+int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr,
+ const char* src, char* dst,
+ int* srcSizePtr, int targetDstSize);
+
+/*! LZ4_setCompressionLevel() : v1.8.0 (experimental)
+ * It's possible to change compression level after LZ4_resetStreamHC(), between 2 invocations of LZ4_compress_HC_continue*(),
+ * but that requires to stay in the same mode (aka 1-10 or 11-12).
+ * This function ensures this condition.
+ */
+void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
+
+
+
+#endif /* LZ4_HC_SLO_098092834 */
+#endif /* LZ4_HC_STATIC_LINKING_ONLY */
diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index d1913fe..e9e54d8 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -1,6 +1,6 @@
/*
lz4opt.h - Optimal Mode of LZ4
- Copyright (C) 2015-2016, Przemyslaw Skibinski <inikep@gmail.com>
+ Copyright (C) 2015-2017, Przemyslaw Skibinski <inikep@gmail.com>
Note : this file is intended to be included within lz4hc.c
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@@ -36,14 +36,12 @@
#define LZ4_OPT_NUM (1<<12)
-typedef struct
-{
+typedef struct {
int off;
int len;
} LZ4HC_match_t;
-typedef struct
-{
+typedef struct {
int price;
int off;
int mlen;
@@ -51,11 +49,12 @@ typedef struct
} LZ4HC_optimal_t;
-/* price in bits */
+/* price in bytes */
FORCE_INLINE size_t LZ4HC_literalsPrice(size_t litlen)
{
- size_t price = 8*litlen;
- if (litlen >= (size_t)RUN_MASK) price+=8*(1+(litlen-RUN_MASK)/255);
+ size_t price = litlen;
+ if (litlen >= (size_t)RUN_MASK)
+ price += 1 + (litlen-RUN_MASK)/255;
return price;
}
@@ -63,12 +62,12 @@ FORCE_INLINE size_t LZ4HC_literalsPrice(size_t litlen)
/* requires mlen >= MINMATCH */
FORCE_INLINE size_t LZ4HC_sequencePrice(size_t litlen, size_t mlen)
{
- size_t price = 16 + 8; /* 16-bit offset + token */
+ size_t price = 2 + 1; /* 16-bit offset + token */
price += LZ4HC_literalsPrice(litlen);
- mlen -= MINMATCH;
- if (mlen >= (size_t)ML_MASK) price+=8*(1+(mlen-ML_MASK)/255);
+ if (mlen >= (size_t)(ML_MASK+MINMATCH))
+ price+= 1 + (mlen-(ML_MASK+MINMATCH))/255;
return price;
}
@@ -123,6 +122,8 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches (
matchLength = LZ4_count(ip, match, vLimit);
if ((ip+matchLength == vLimit) && (vLimit < iHighLimit))
matchLength += LZ4_count(ip+matchLength, base+dictLimit, iHighLimit);
+ if (matchIndex+matchLength >= dictLimit)
+ match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
}
if (matchLength > best_mlen) {
@@ -141,6 +142,8 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches (
if (ip+matchLength >= iHighLimit) /* equal : no way to know if inf or sup */
break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */
+ DEBUGLOG(6, "ip :%016llX", (U64)ip);
+ DEBUGLOG(6, "match:%016llX", (U64)match);
if (*(ip+matchLength) < *(match+matchLength)) {
*ptr0 = delta0;
ptr0 = &DELTANEXTMAXD(matchIndex*2);
@@ -192,13 +195,13 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches (
}
-#define SET_PRICE(pos, mlen, offset, litlen, price) \
+#define SET_PRICE(pos, ml, offset, ll, cost) \
{ \
while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \
- opt[pos].mlen = (int)mlen; \
+ opt[pos].mlen = (int)ml; \
opt[pos].off = (int)offset; \
- opt[pos].litlen = (int)litlen; \
- opt[pos].price = (int)price; \
+ opt[pos].litlen = (int)ll; \
+ opt[pos].price = (int)cost; \
}
@@ -209,15 +212,12 @@ static int LZ4HC_compress_optimal (
int inputSize,
int maxOutputSize,
limitedOutput_directive limit,
- const size_t sufficient_len,
+ size_t sufficient_len,
const int fullUpdate
)
{
- LZ4HC_optimal_t opt[LZ4_OPT_NUM + 1];
+ LZ4HC_optimal_t opt[LZ4_OPT_NUM + 1]; /* this uses a bit too much stack memory to my taste ... */
LZ4HC_match_t matches[LZ4_OPT_NUM + 1];
- const BYTE *inr = NULL;
- size_t res, cur, cur2;
- size_t i, llen, litlen, mlen, best_mlen, price, offset, best_off, match_num, last_pos;
const BYTE* ip = (const BYTE*) source;
const BYTE* anchor = ip;
@@ -228,18 +228,23 @@ static int LZ4HC_compress_optimal (
BYTE* const oend = op + maxOutputSize;
/* init */
+ DEBUGLOG(5, "LZ4HC_compress_optimal");
+ if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1;
ctx->end += inputSize;
ip++;
/* Main Loop */
while (ip < mflimit) {
- memset(opt, 0, sizeof(LZ4HC_optimal_t));
- last_pos = 0;
- llen = ip - anchor;
+ size_t const llen = ip - anchor;
+ size_t last_pos = 0;
+ size_t match_num, cur, best_mlen, best_off;
+ memset(opt, 0, sizeof(LZ4HC_optimal_t)); /* memset only the first one */
+
match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate);
if (!match_num) { ip++; continue; }
if ((size_t)matches[match_num-1].len > sufficient_len) {
+ /* good enough solution : immediate encoding */
best_mlen = matches[match_num-1].len;
best_off = matches[match_num-1].off;
cur = 0;
@@ -248,45 +253,46 @@ static int LZ4HC_compress_optimal (
}
/* set prices using matches at position = 0 */
- for (i = 0; i < match_num; i++) {
- mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH;
- best_mlen = (matches[i].len < LZ4_OPT_NUM) ? matches[i].len : LZ4_OPT_NUM;
- while (mlen <= best_mlen) {
- litlen = 0;
- price = LZ4HC_sequencePrice(llen + litlen, mlen) - LZ4HC_literalsPrice(llen);
- SET_PRICE(mlen, mlen, matches[i].off, litlen, price);
- mlen++;
- }
- }
+ { size_t matchNb;
+ for (matchNb = 0; matchNb < match_num; matchNb++) {
+ size_t mlen = (matchNb>0) ? (size_t)matches[matchNb-1].len+1 : MINMATCH;
+ best_mlen = matches[matchNb].len; /* necessarily < sufficient_len < LZ4_OPT_NUM */
+ for ( ; mlen <= best_mlen ; mlen++) {
+ size_t const cost = LZ4HC_sequencePrice(llen, mlen) - LZ4HC_literalsPrice(llen);
+ SET_PRICE(mlen, mlen, matches[matchNb].off, 0, cost); /* updates last_pos and opt[pos] */
+ } } }
- if (last_pos < MINMATCH) { ip++; continue; }
+ if (last_pos < MINMATCH) { ip++; continue; } /* note : on clang at least, this test improves performance */
/* check further positions */
opt[0].mlen = opt[1].mlen = 1;
for (cur = 1; cur <= last_pos; cur++) {
- inr = ip + cur;
-
- if (opt[cur-1].mlen == 1) {
- litlen = opt[cur-1].litlen + 1;
- if (cur != litlen) {
- price = opt[cur - litlen].price + LZ4HC_literalsPrice(litlen);
+ const BYTE* const curPtr = ip + cur;
+
+ /* establish baseline price if cur is literal */
+ { size_t price, litlen;
+ if (opt[cur-1].mlen == 1) {
+ /* no match at previous position */
+ litlen = opt[cur-1].litlen + 1;
+ if (cur > litlen) {
+ price = opt[cur - litlen].price + LZ4HC_literalsPrice(litlen);
+ } else {
+ price = LZ4HC_literalsPrice(llen + litlen) - LZ4HC_literalsPrice(llen);
+ }
} else {
- price = LZ4HC_literalsPrice(llen + litlen) - LZ4HC_literalsPrice(llen);
+ litlen = 1;
+ price = opt[cur - 1].price + LZ4HC_literalsPrice(1);
}
- } else {
- litlen = 1;
- price = opt[cur - 1].price + LZ4HC_literalsPrice(litlen);
- }
- mlen = 1;
- best_mlen = 0;
- if (cur > last_pos || price < (size_t)opt[cur].price)
- SET_PRICE(cur, mlen, best_mlen, litlen, price);
+ if (price < (size_t)opt[cur].price)
+ SET_PRICE(cur, 1 /*mlen*/, 0 /*off*/, litlen, price); /* note : increases last_pos */
+ }
- if (cur == last_pos || inr >= mflimit) break;
+ if (cur == last_pos || curPtr >= mflimit) break;
- match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches, fullUpdate);
- if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) {
+ match_num = LZ4HC_BinTree_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
+ if ((match_num > 0) && (size_t)matches[match_num-1].len > sufficient_len) {
+ /* immediate encoding */
best_mlen = matches[match_num-1].len;
best_off = matches[match_num-1].off;
last_pos = cur + 1;
@@ -294,60 +300,57 @@ static int LZ4HC_compress_optimal (
}
/* set prices using matches at position = cur */
- for (i = 0; i < match_num; i++) {
- mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH;
- cur2 = cur;
- best_mlen = (cur2 + matches[i].len < LZ4_OPT_NUM) ? (size_t)matches[i].len : LZ4_OPT_NUM - cur2;
-
- while (mlen <= best_mlen) {
- if (opt[cur2].mlen == 1) {
- litlen = opt[cur2].litlen;
-
- if (cur2 != litlen)
- price = opt[cur2 - litlen].price + LZ4HC_sequencePrice(litlen, mlen);
- else
- price = LZ4HC_sequencePrice(llen + litlen, mlen) - LZ4HC_literalsPrice(llen);
- } else {
- litlen = 0;
- price = opt[cur2].price + LZ4HC_sequencePrice(litlen, mlen);
- }
-
- if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) { // || (((int)price == opt[cur2 + mlen].price) && (opt[cur2 + mlen-1].mlen == 1))) {
- SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price);
- }
- mlen++;
- }
- }
+ { size_t matchNb;
+ for (matchNb = 0; matchNb < match_num; matchNb++) {
+ size_t ml = (matchNb>0) ? (size_t)matches[matchNb-1].len+1 : MINMATCH;
+ best_mlen = (cur + matches[matchNb].len < LZ4_OPT_NUM) ?
+ (size_t)matches[matchNb].len : LZ4_OPT_NUM - cur;
+
+ for ( ; ml <= best_mlen ; ml++) {
+ size_t ll, price;
+ if (opt[cur].mlen == 1) {
+ ll = opt[cur].litlen;
+ if (cur > ll)
+ price = opt[cur - ll].price + LZ4HC_sequencePrice(ll, ml);
+ else
+ price = LZ4HC_sequencePrice(llen + ll, ml) - LZ4HC_literalsPrice(llen);
+ } else {
+ ll = 0;
+ price = opt[cur].price + LZ4HC_sequencePrice(0, ml);
+ }
+
+ if (cur + ml > last_pos || price < (size_t)opt[cur + ml].price) {
+ SET_PRICE(cur + ml, ml, matches[matchNb].off, ll, price);
+ } } } }
} /* for (cur = 1; cur <= last_pos; cur++) */
best_mlen = opt[last_pos].mlen;
best_off = opt[last_pos].off;
cur = last_pos - best_mlen;
-encode: /* cur, last_pos, best_mlen, best_off have to be set */
+encode: /* cur, last_pos, best_mlen, best_off must be set */
opt[0].mlen = 1;
- while (1) {
- mlen = opt[cur].mlen;
- offset = opt[cur].off;
+ while (1) { /* from end to beginning */
+ size_t const ml = opt[cur].mlen;
+ int const offset = opt[cur].off;
opt[cur].mlen = (int)best_mlen;
opt[cur].off = (int)best_off;
- best_mlen = mlen;
+ best_mlen = ml;
best_off = offset;
- if (mlen > cur) break;
- cur -= mlen;
+ if (ml > cur) break; /* can this happen ? */
+ cur -= ml;
}
+ /* encode all recorded sequences */
cur = 0;
while (cur < last_pos) {
- mlen = opt[cur].mlen;
- if (mlen == 1) { ip++; cur++; continue; }
- offset = opt[cur].off;
- cur += mlen;
-
- res = LZ4HC_encodeSequence(&ip, &op, &anchor, (int)mlen, ip - offset, limit, oend);
- if (res) return 0;
+ int const ml = opt[cur].mlen;
+ int const offset = opt[cur].off;
+ if (ml == 1) { ip++; cur++; continue; }
+ cur += ml;
+ if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) ) return 0;
}
- }
+ } /* while (ip < mflimit) */
/* Encode Last Literals */
{ int lastRun = (int)(iend - anchor);
diff --git a/lib/xxhash.c b/lib/xxhash.c
index e9ff2d4..a532358 100644
--- a/lib/xxhash.c
+++ b/lib/xxhash.c
@@ -113,19 +113,24 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp
***************************************/
#ifdef _MSC_VER /* Visual Studio */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
-# define FORCE_INLINE static __forceinline
-#else
-# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
-# ifdef __GNUC__
-# define FORCE_INLINE static inline __attribute__((always_inline))
-# else
-# define FORCE_INLINE static inline
-# endif
-# else
-# define FORCE_INLINE static
-# endif /* __STDC_VERSION__ */
#endif
+#ifndef FORCE_INLINE
+# ifdef _MSC_VER /* Visual Studio */
+# define FORCE_INLINE static __forceinline
+# else
+# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
+# ifdef __GNUC__
+# define FORCE_INLINE static inline __attribute__((always_inline))
+# else
+# define FORCE_INLINE static inline
+# endif
+# else
+# define FORCE_INLINE static
+# endif /* __STDC_VERSION__ */
+# endif /* _MSC_VER */
+#endif /* FORCE_INLINE */
+
/* *************************************
* Basic Types
generated by cgit v0.12 at 2024-11-05 14:49:42 (GMT)