summaryrefslogtreecommitdiffstats
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/Makefile28
-rw-r--r--lib/README.md67
-rw-r--r--lib/dll/liblz4.def62
-rw-r--r--lib/lz4.c859
-rw-r--r--lib/lz4.h368
-rw-r--r--lib/lz4frame.c420
-rw-r--r--lib/lz4frame.h143
-rw-r--r--lib/lz4hc.c351
-rw-r--r--lib/lz4hc.h244
-rw-r--r--lib/xxhash.c614
-rw-r--r--lib/xxhash.h221
11 files changed, 2089 insertions, 1288 deletions
diff --git a/lib/Makefile b/lib/Makefile
index 88d9b4f..cb1571c 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -71,7 +71,11 @@ else
SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
endif
+ifneq (,$(filter Windows%,$(OS)))
+LIBLZ4 = liblz4-$(LIBVER_MAJOR)
+else
LIBLZ4 = liblz4.$(SHARED_EXT_VER)
+endif
.PHONY: default
default: lib-release
@@ -103,8 +107,7 @@ $(LIBLZ4): $(SRCFILES)
ifeq ($(BUILD_SHARED),yes) # can be disabled on command line
@echo compiling dynamic library $(LIBVER)
ifneq (,$(filter Windows%,$(OS)))
- $(Q)$(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll\$@.dll
- dlltool -D dll\liblz4.dll -d dll\liblz4.def -l dll\liblz4.lib
+ $(Q)$(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll/$@.dll -Wl,--out-implib,dll/liblz4.lib
else
$(Q)$(CC) $(FLAGS) -shared $^ -fPIC -fvisibility=hidden $(SONAME_FLAGS) -o $@
@echo creating versioned links
@@ -124,7 +127,7 @@ clean:
#-----------------------------------------------------------------------------
# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
#-----------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku MidnightBSD))
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku MidnightBSD MINGW32_NT-6.1 MINGW64_NT-6.1 MINGW32_NT-10.0 MINGW64_NT-10.0))
.PHONY: listL120
listL120: # extract lines >= 120 characters in *.{c,h}, by Takayuki Matsuoka (note : $$, for Makefile compatibility)
@@ -138,12 +141,14 @@ PREFIX ?= /usr/local
prefix ?= $(PREFIX)
EXEC_PREFIX ?= $(prefix)
exec_prefix ?= $(EXEC_PREFIX)
+BINDIR ?= $(exec_prefix)/bin
+bindir ?= $(BINDIR)
LIBDIR ?= $(exec_prefix)/lib
libdir ?= $(LIBDIR)
INCLUDEDIR ?= $(prefix)/include
includedir ?= $(INCLUDEDIR)
-ifneq (,$(filter $(OS),OpenBSD FreeBSD NetBSD DragonFly))
+ifneq (,$(filter $(OS),OpenBSD FreeBSD NetBSD DragonFly MidnightBSD))
PKGCONFIGDIR ?= $(prefix)/libdata/pkgconfig
else
PKGCONFIGDIR ?= $(libdir)/pkgconfig
@@ -168,7 +173,7 @@ liblz4.pc: liblz4.pc.in Makefile
$< >$@
install: lib liblz4.pc
- $(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(pkgconfigdir)/ $(DESTDIR)$(includedir)/ $(DESTDIR)$(libdir)/
+ $(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(pkgconfigdir)/ $(DESTDIR)$(includedir)/ $(DESTDIR)$(libdir)/ $(DESTDIR)$(bindir)/
$(Q)$(INSTALL_DATA) liblz4.pc $(DESTDIR)$(pkgconfigdir)/
@echo Installing libraries
ifeq ($(BUILD_STATIC),yes)
@@ -176,10 +181,18 @@ ifeq ($(BUILD_STATIC),yes)
$(Q)$(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(includedir)/lz4frame_static.h
endif
ifeq ($(BUILD_SHARED),yes)
+# Traditionnally, one installs the DLLs in the bin directory as programs
+# search them first in their directory. This allows to not pollute system
+# directories (like c:/windows/system32), nor modify the PATH variable.
+ifneq (,$(filter Windows%,$(OS)))
+ $(Q)$(INSTALL_PROGRAM) dll/$(LIBLZ4).dll $(DESTDIR)$(bindir)
+ $(Q)$(INSTALL_PROGRAM) dll/liblz4.lib $(DESTDIR)$(libdir)
+else
$(Q)$(INSTALL_PROGRAM) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)
$(Q)ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_MAJOR)
$(Q)ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT)
endif
+endif
@echo Installing headers in $(includedir)
$(Q)$(INSTALL_DATA) lz4.h $(DESTDIR)$(includedir)/lz4.h
$(Q)$(INSTALL_DATA) lz4hc.h $(DESTDIR)$(includedir)/lz4hc.h
@@ -188,9 +201,14 @@ endif
uninstall:
$(Q)$(RM) $(DESTDIR)$(pkgconfigdir)/liblz4.pc
+ifneq (,$(filter Windows%,$(OS)))
+ $(Q)$(RM) $(DESTDIR)$(bindir)/$(LIBLZ4).dll
+ $(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.lib
+else
$(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT)
$(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_MAJOR)
$(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_VER)
+endif
$(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.a
$(Q)$(RM) $(DESTDIR)$(includedir)/lz4.h
$(Q)$(RM) $(DESTDIR)$(includedir)/lz4hc.h
diff --git a/lib/README.md b/lib/README.md
index 7082fe3..b753195 100644
--- a/lib/README.md
+++ b/lib/README.md
@@ -7,8 +7,8 @@ not all of them are necessary.
#### Minimal LZ4 build
The minimum required is **`lz4.c`** and **`lz4.h`**,
-which provides the fast compression and decompression algorithm.
-They generate and decode data using [LZ4 block format].
+which provides the fast compression and decompression algorithms.
+They generate and decode data using the [LZ4 block format].
#### High Compression variant
@@ -16,13 +16,14 @@ They generate and decode data using [LZ4 block format].
For more compression ratio at the cost of compression speed,
the High Compression variant called **lz4hc** is available.
Add files **`lz4hc.c`** and **`lz4hc.h`**.
-The variant still depends on regular `lib/lz4.*` source files.
+This variant also compresses data using the [LZ4 block format],
+and depends on regular `lib/lz4.*` source files.
-#### Frame variant, for interoperability
+#### Frame support, for interoperability
In order to produce compressed data compatible with `lz4` command line utility,
-it's necessary to encode lz4-compressed blocks using the [official interoperable frame format].
+it's necessary to use the [official interoperable frame format].
This format is generated and decoded automatically by the **lz4frame** library.
Its public API is described in `lib/lz4frame.h`.
In order to work properly, lz4frame needs all other modules present in `/lib`,
@@ -32,15 +33,63 @@ So it's necessary to include all `*.c` and `*.h` files present in `/lib`.
#### Advanced / Experimental API
-A complex API defined in `lz4frame_static.h` contains definitions
-which are not guaranteed to remain stable in future versions.
-As a consequence, it must be used with static linking ***only***.
+Definitions which are not guaranteed to remain stable in future versions,
+are protected behind macros, such as `LZ4_STATIC_LINKING_ONLY`.
+As the name implies, these definitions can only be invoked
+in the context of static linking ***only***.
+Otherwise, dependent application may fail on API or ABI break in the future.
+The associated symbols are also not present in dynamic library by default.
+Should they be nonetheless needed, it's possible to force their publication
+by using build macro `LZ4_PUBLISH_STATIC_FUNCTIONS`.
+
+
+#### Build macros
+
+The following build macro can be selected at compilation time :
+
+- `LZ4_FAST_DEC_LOOP` : this triggers the optimized decompression loop.
+ This loops works great on x86/x64 cpus, and is automatically enabled on this platform.
+ It's possible to enable or disable it manually, by passing `LZ4_FAST_DEC_LOOP=1` or `0` to the preprocessor.
+ For example, with `gcc` : `-DLZ4_FAST_DEC_LOOP=1`,
+ and with `make` : `CPPFLAGS+=-DLZ4_FAST_DEC_LOOP=1 make lz4`.
+
+- `LZ4_DISTANCE_MAX` : control the maximum offset that the compressor will allow.
+ Set to 65535 by default, which is the maximum value supported by lz4 format.
+ Reducing maximum distance will reduce opportunities for LZ4 to find matches,
+ hence will produce worse the compression ratio.
+ However, a smaller max distance may allow compatibility with specific decoders using limited memory budget.
+ This build macro only influences the compressed output of the compressor.
+
+- `LZ4_DISABLE_DEPRECATE_WARNINGS` : invoking a deprecated function will make the compiler generate a warning.
+ This is meant to invite users to update their source code.
+ Should this be a problem, it's generally possible to make the compiler ignore these warnings,
+ for example with `-Wno-deprecated-declarations` on `gcc`,
+ or `_CRT_SECURE_NO_WARNINGS` for Visual Studio.
+ Another method is to define `LZ4_DISABLE_DEPRECATE_WARNINGS`
+ before including the LZ4 header files.
+
+
+#### Amalgamation
+
+lz4 source code can be amalgamated into a single file.
+One can combine all source code into `lz4_all.c` by using following command:
+```
+cat lz4.c > lz4_all.c
+cat lz4hc.c >> lz4_all.c
+cat lz4frame.c >> lz4_all.c
+```
+(`cat` file order is important) then compile `lz4_all.c`.
+All `*.h` files present in `/lib` remain necessary to compile `lz4_all.c`.
#### Windows : using MinGW+MSYS to create DLL
DLL can be created using MinGW+MSYS with the `make liblz4` command.
This command creates `dll\liblz4.dll` and the import library `dll\liblz4.lib`.
+To override the `dlltool` command when cross-compiling on Linux, just set the `DLLTOOL` variable. Example of cross compilation on Linux with mingw-w64 64 bits:
+```
+make BUILD_STATIC=no CC=x86_64-w64-mingw32-gcc DLLTOOL=x86_64-w64-mingw32-dlltool OS=Windows_NT
+```
The import library is only required with Visual C++.
The header files `lz4.h`, `lz4hc.h`, `lz4frame.h` and the dynamic library
`dll\liblz4.dll` are required to compile a project using gcc/MinGW.
@@ -48,7 +97,7 @@ The dynamic library has to be added to linking options.
It means that if a project that uses LZ4 consists of a single `test-dll.c`
file it should be linked with `dll\liblz4.dll`. For example:
```
- gcc $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\liblz4.dll
+ $(CC) $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\liblz4.dll
```
The compiled executable will require LZ4 DLL which is available at `dll\liblz4.dll`.
diff --git a/lib/dll/liblz4.def b/lib/dll/liblz4.def
deleted file mode 100644
index 0ace223..0000000
--- a/lib/dll/liblz4.def
+++ /dev/null
@@ -1,62 +0,0 @@
-LIBRARY liblz4.dll
-EXPORTS
- LZ4F_compressBegin
- LZ4F_compressBound
- LZ4F_compressEnd
- LZ4F_compressFrame
- LZ4F_compressFrameBound
- LZ4F_compressUpdate
- LZ4F_createCompressionContext
- LZ4F_createDecompressionContext
- LZ4F_decompress
- LZ4F_flush
- LZ4F_freeCompressionContext
- LZ4F_freeDecompressionContext
- LZ4F_getErrorName
- LZ4F_getFrameInfo
- LZ4F_getVersion
- LZ4F_isError
- LZ4_compress
- LZ4_compressBound
- LZ4_compressHC
- LZ4_compressHC_continue
- LZ4_compressHC_limitedOutput
- LZ4_compressHC_limitedOutput_continue
- LZ4_compressHC_limitedOutput_withStateHC
- LZ4_compressHC_withStateHC
- LZ4_compress_HC
- LZ4_compress_HC_continue
- LZ4_compress_HC_extStateHC
- LZ4_compress_continue
- LZ4_compress_default
- LZ4_compress_destSize
- LZ4_compress_fast
- LZ4_compress_fast_continue
- LZ4_compress_fast_extState
- LZ4_compress_limitedOutput
- LZ4_compress_limitedOutput_continue
- LZ4_compress_limitedOutput_withState
- LZ4_compress_withState
- LZ4_createStream
- LZ4_createStreamDecode
- LZ4_createStreamHC
- LZ4_decompress_fast
- LZ4_decompress_fast_continue
- LZ4_decompress_fast_usingDict
- LZ4_decompress_safe
- LZ4_decompress_safe_continue
- LZ4_decompress_safe_partial
- LZ4_decompress_safe_usingDict
- LZ4_freeStream
- LZ4_freeStreamDecode
- LZ4_freeStreamHC
- LZ4_loadDict
- LZ4_loadDictHC
- LZ4_resetStream
- LZ4_resetStreamHC
- LZ4_saveDict
- LZ4_saveDictHC
- LZ4_setStreamDecode
- LZ4_sizeofState
- LZ4_sizeofStateHC
- LZ4_versionNumber
diff --git a/lib/lz4.c b/lib/lz4.c
index 4046102..693bdaf 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -32,7 +32,6 @@
- LZ4 source repository : https://github.com/lz4/lz4
*/
-
/*-************************************
* Tuning parameters
**************************************/
@@ -91,6 +90,14 @@
/*-************************************
* Dependency
**************************************/
+/*
+ * LZ4_SRC_INCLUDED:
+ * Amalgamation flag, whether lz4.c is included
+ */
+#ifndef LZ4_SRC_INCLUDED
+# define LZ4_SRC_INCLUDED 1
+#endif
+
#define LZ4_STATIC_LINKING_ONLY
#define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */
#include "lz4.h"
@@ -136,7 +143,7 @@
* and also LZ4_wildCopy is forcibly inlined, so that the O2 attribute
* of LZ4_wildCopy does not affect the compression speed.
*/
-#if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__)
+#if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) && !defined(__clang__)
# define LZ4_FORCE_O2_GCC_PPC64LE __attribute__((optimize("O2")))
# define LZ4_FORCE_O2_INLINE_GCC_PPC64LE __attribute__((optimize("O2"))) LZ4_FORCE_INLINE
#else
@@ -170,7 +177,7 @@
/*-************************************
-* Basic Types
+* Types
**************************************/
#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# include <stdint.h>
@@ -195,6 +202,13 @@
typedef size_t reg_t; /* 32-bits in x32 mode */
#endif
+typedef enum {
+ notLimited = 0,
+ limitedOutput = 1,
+ fillOutput = 2
+} limitedOutput_directive;
+
+
/*-************************************
* Reading and writing into memory
**************************************/
@@ -228,7 +242,7 @@ static reg_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArc
static void LZ4_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
-#else /* safe and portable access through memcpy() */
+#else /* safe and portable access using memcpy() */
static U16 LZ4_read16(const void* memPtr)
{
@@ -290,6 +304,86 @@ void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd)
do { memcpy(d,s,8); d+=8; s+=8; } while (d<e);
}
+static const unsigned inc32table[8] = {0, 1, 2, 1, 0, 4, 4, 4};
+static const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3};
+
+
+#ifndef LZ4_FAST_DEC_LOOP
+# if defined(__i386__) || defined(__x86_64__)
+# define LZ4_FAST_DEC_LOOP 1
+# else
+# define LZ4_FAST_DEC_LOOP 0
+# endif
+#endif
+
+#if LZ4_FAST_DEC_LOOP
+
+LZ4_FORCE_O2_INLINE_GCC_PPC64LE void
+LZ4_memcpy_using_offset_base(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset)
+{
+ if (offset < 8) {
+ dstPtr[0] = srcPtr[0];
+ dstPtr[1] = srcPtr[1];
+ dstPtr[2] = srcPtr[2];
+ dstPtr[3] = srcPtr[3];
+ srcPtr += inc32table[offset];
+ memcpy(dstPtr+4, srcPtr, 4);
+ srcPtr -= dec64table[offset];
+ dstPtr += 8;
+ } else {
+ memcpy(dstPtr, srcPtr, 8);
+ dstPtr += 8;
+ srcPtr += 8;
+ }
+
+ LZ4_wildCopy(dstPtr, srcPtr, dstEnd);
+}
+
+/* customized variant of memcpy, which can overwrite up to 32 bytes beyond dstEnd
+ * this version copies two times 16 bytes (instead of one time 32 bytes)
+ * because it must be compatible with offsets >= 16. */
+LZ4_FORCE_O2_INLINE_GCC_PPC64LE void
+LZ4_wildCopy32(void* dstPtr, const void* srcPtr, void* dstEnd)
+{
+ BYTE* d = (BYTE*)dstPtr;
+ const BYTE* s = (const BYTE*)srcPtr;
+ BYTE* const e = (BYTE*)dstEnd;
+
+ do { memcpy(d,s,16); memcpy(d+16,s+16,16); d+=32; s+=32; } while (d<e);
+}
+
+LZ4_FORCE_O2_INLINE_GCC_PPC64LE void
+LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset)
+{
+ BYTE v[8];
+ switch(offset) {
+ case 1:
+ memset(v, *srcPtr, 8);
+ goto copy_loop;
+ case 2:
+ memcpy(v, srcPtr, 2);
+ memcpy(&v[2], srcPtr, 2);
+ memcpy(&v[4], &v[0], 4);
+ goto copy_loop;
+ case 4:
+ memcpy(v, srcPtr, 4);
+ memcpy(&v[4], srcPtr, 4);
+ goto copy_loop;
+ default:
+ LZ4_memcpy_using_offset_base(dstPtr, srcPtr, dstEnd, offset);
+ return;
+ }
+
+ copy_loop:
+ memcpy(dstPtr, v, 8);
+ dstPtr += 8;
+ while (dstPtr < dstEnd) {
+ memcpy(dstPtr, v, 8);
+ dstPtr += 8;
+ }
+}
+#endif
+
/*-************************************
* Common Constants
@@ -300,14 +394,20 @@ void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd)
#define LASTLITERALS 5 /* see ../doc/lz4_Block_format.md#parsing-restrictions */
#define MFLIMIT 12 /* see ../doc/lz4_Block_format.md#parsing-restrictions */
#define MATCH_SAFEGUARD_DISTANCE ((2*WILDCOPYLENGTH) - MINMATCH) /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */
+#define FASTLOOP_SAFE_DISTANCE 64
static const int LZ4_minLength = (MFLIMIT+1);
#define KB *(1 <<10)
#define MB *(1 <<20)
#define GB *(1U<<30)
-#define MAXD_LOG 16
-#define MAX_DISTANCE ((1 << MAXD_LOG) - 1)
+#ifndef LZ4_DISTANCE_MAX /* can be user - defined at compile time */
+# define LZ4_DISTANCE_MAX 65535
+#endif
+
+#if (LZ4_DISTANCE_MAX > 65535) /* max supported by LZ4 format */
+# error "LZ4_DISTANCE_MAX is too big : must be <= 65535"
+#endif
#define ML_BITS 4
#define ML_MASK ((1U<<ML_BITS)-1)
@@ -455,7 +555,6 @@ static const U32 LZ4_skipTrigger = 6; /* Increase this value ==> compression ru
/*-************************************
* Local Structures and types
**************************************/
-typedef enum { notLimited = 0, limitedOutput = 1, fillOutput = 2 } limitedOutput_directive;
typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t;
/**
@@ -522,13 +621,14 @@ static U32 LZ4_hash4(U32 sequence, tableType_t const tableType)
static U32 LZ4_hash5(U64 sequence, tableType_t const tableType)
{
- static const U64 prime5bytes = 889523592379ULL;
- static const U64 prime8bytes = 11400714785074694791ULL;
const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG;
- if (LZ4_isLittleEndian())
+ if (LZ4_isLittleEndian()) {
+ const U64 prime5bytes = 889523592379ULL;
return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog));
- else
+ } else {
+ const U64 prime8bytes = 11400714785074694791ULL;
return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog));
+ }
}
LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType)
@@ -609,6 +709,15 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
LZ4_stream_t_internal* const cctx,
const int inputSize,
const tableType_t tableType) {
+ /* If compression failed during the previous step, then the context
+ * is marked as dirty, therefore, it has to be fully reset.
+ */
+ if (cctx->dirty) {
+ DEBUGLOG(5, "LZ4_prepareTable: Full reset for %p", cctx);
+ MEM_INIT(cctx, 0, sizeof(LZ4_stream_t_internal));
+ return;
+ }
+
/* If the table hasn't been used, it's guaranteed to be zeroed out, and is
* therefore safe to use no matter what mode we're in. Otherwise, we figure
* out if it's safe to leave as is or whether it needs to be reset.
@@ -629,7 +738,7 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
}
}
- /* Adding a gap, so all previous entries are > MAX_DISTANCE back, is faster
+ /* Adding a gap, so all previous entries are > LZ4_DISTANCE_MAX back, is faster
* than compressing without a gap. However, compressing with
* currentOffset == 0 is faster still, so we preserve that case.
*/
@@ -651,14 +760,15 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
const char* const source,
char* const dest,
const int inputSize,
- int *inputConsumed, /* only written when outputLimited == fillOutput */
+ int *inputConsumed, /* only written when outputDirective == fillOutput */
const int maxOutputSize,
- const limitedOutput_directive outputLimited,
+ const limitedOutput_directive outputDirective,
const tableType_t tableType,
const dict_directive dictDirective,
const dictIssue_directive dictIssue,
- const U32 acceleration)
+ const int acceleration)
{
+ int result;
const BYTE* ip = (const BYTE*) source;
U32 const startIndex = cctx->currentOffset;
@@ -693,9 +803,10 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
U32 forwardH;
DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, tableType=%u", inputSize, tableType);
- /* Init conditions */
- if (outputLimited == fillOutput && maxOutputSize < 1) return 0; /* Impossible to store anything */
- if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */
+ /* If init conditions are not met, we don't have to mark stream
+ * as having dirty context, since no action was taken yet */
+ if (outputDirective == fillOutput && maxOutputSize < 1) return 0; /* Impossible to store anything */
+ if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */
if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */
if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */
assert(acceleration >= 1);
@@ -728,8 +839,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
/* Find a match */
if (tableType == byPtr) {
const BYTE* forwardIp = ip;
- unsigned step = 1;
- unsigned searchMatchNb = acceleration << LZ4_skipTrigger;
+ int step = 1;
+ int searchMatchNb = acceleration << LZ4_skipTrigger;
do {
U32 const h = forwardH;
ip = forwardIp;
@@ -743,14 +854,14 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
forwardH = LZ4_hashPosition(forwardIp, tableType);
LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base);
- } while ( (match+MAX_DISTANCE < ip)
+ } while ( (match+LZ4_DISTANCE_MAX < ip)
|| (LZ4_read32(match) != LZ4_read32(ip)) );
} else { /* byU32, byU16 */
const BYTE* forwardIp = ip;
- unsigned step = 1;
- unsigned searchMatchNb = acceleration << LZ4_skipTrigger;
+ int step = 1;
+ int searchMatchNb = acceleration << LZ4_skipTrigger;
do {
U32 const h = forwardH;
U32 const current = (U32)(forwardIp - base);
@@ -794,8 +905,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue; /* match outside of valid area */
assert(matchIndex < current);
- if ((tableType != byU16) && (matchIndex+MAX_DISTANCE < current)) continue; /* too far */
- if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE); /* too_far presumed impossible with byU16 */
+ if ((tableType != byU16) && (matchIndex+LZ4_DISTANCE_MAX < current)) continue; /* too far */
+ if (tableType == byU16) assert((current - matchIndex) <= LZ4_DISTANCE_MAX); /* too_far presumed impossible with byU16 */
if (LZ4_read32(match) == LZ4_read32(ip)) {
if (maybe_extMem) offset = current - matchIndex;
@@ -811,16 +922,17 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
/* Encode Literals */
{ unsigned const litLength = (unsigned)(ip - anchor);
token = op++;
- if ((outputLimited == limitedOutput) && /* Check output buffer overflow */
- (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)))
- return 0;
- if ((outputLimited == fillOutput) &&
+ if ((outputDirective == limitedOutput) && /* Check output buffer overflow */
+ (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)) )
+ return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */
+
+ if ((outputDirective == fillOutput) &&
(unlikely(op + (litLength+240)/255 /* litlen */ + litLength /* literals */ + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit))) {
op--;
goto _last_literals;
}
if (litLength >= RUN_MASK) {
- int len = (int)litLength-RUN_MASK;
+ int len = (int)(litLength - RUN_MASK);
*token = (RUN_MASK<<ML_BITS);
for(; len >= 255 ; len-=255) *op++ = 255;
*op++ = (BYTE)len;
@@ -843,7 +955,7 @@ _next_match:
* - token and *token : position to write 4-bits for match length; higher 4-bits for literal length supposed already written
*/
- if ((outputLimited == fillOutput) &&
+ if ((outputDirective == fillOutput) &&
(op + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit)) {
/* the match was too close to the end, rewind and go to last literals */
op = token;
@@ -853,11 +965,11 @@ _next_match:
/* Encode Offset */
if (maybe_extMem) { /* static test */
DEBUGLOG(6, " with offset=%u (ext if > %i)", offset, (int)(ip - (const BYTE*)source));
- assert(offset <= MAX_DISTANCE && offset > 0);
+ assert(offset <= LZ4_DISTANCE_MAX && offset > 0);
LZ4_writeLE16(op, (U16)offset); op+=2;
} else {
DEBUGLOG(6, " with offset=%u (same segment)", (U32)(ip - match));
- assert(ip-match <= MAX_DISTANCE);
+ assert(ip-match <= LZ4_DISTANCE_MAX);
LZ4_writeLE16(op, (U16)(ip - match)); op+=2;
}
@@ -870,7 +982,7 @@ _next_match:
assert(dictEnd > match);
if (limit > matchlimit) limit = matchlimit;
matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit);
- ip += MINMATCH + matchCode;
+ ip += (size_t)matchCode + MINMATCH;
if (ip==limit) {
unsigned const more = LZ4_count(limit, (const BYTE*)source, matchlimit);
matchCode += more;
@@ -879,19 +991,20 @@ _next_match:
DEBUGLOG(6, " with matchLength=%u starting in extDict", matchCode+MINMATCH);
} else {
matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit);
- ip += MINMATCH + matchCode;
+ ip += (size_t)matchCode + MINMATCH;
DEBUGLOG(6, " with matchLength=%u", matchCode+MINMATCH);
}
- if ((outputLimited) && /* Check output buffer overflow */
+ if ((outputDirective) && /* Check output buffer overflow */
(unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) {
- if (outputLimited == limitedOutput)
- return 0;
- if (outputLimited == fillOutput) {
+ if (outputDirective == fillOutput) {
/* Match description too long : reduce it */
U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 2 - 1 - LASTLITERALS) * 255;
ip -= matchCode - newMatchCode;
matchCode = newMatchCode;
+ } else {
+ assert(outputDirective == limitedOutput);
+ return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */
}
}
if (matchCode >= ML_MASK) {
@@ -922,7 +1035,7 @@ _next_match:
match = LZ4_getPosition(ip, cctx->hashTable, tableType, base);
LZ4_putPosition(ip, cctx->hashTable, tableType, base);
- if ( (match+MAX_DISTANCE >= ip)
+ if ( (match+LZ4_DISTANCE_MAX >= ip)
&& (LZ4_read32(match) == LZ4_read32(ip)) )
{ token=op++; *token=0; goto _next_match; }
@@ -957,7 +1070,7 @@ _next_match:
LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
assert(matchIndex < current);
if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1)
- && ((tableType==byU16) ? 1 : (matchIndex+MAX_DISTANCE >= current))
+ && ((tableType==byU16) ? 1 : (matchIndex+LZ4_DISTANCE_MAX >= current))
&& (LZ4_read32(match) == LZ4_read32(ip)) ) {
token=op++;
*token=0;
@@ -976,15 +1089,17 @@ _next_match:
_last_literals:
/* Encode Last Literals */
{ size_t lastRun = (size_t)(iend - anchor);
- if ( (outputLimited) && /* Check output buffer overflow */
+ if ( (outputDirective) && /* Check output buffer overflow */
(op + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > olimit)) {
- if (outputLimited == fillOutput) {
+ if (outputDirective == fillOutput) {
/* adapt lastRun to fill 'dst' */
- lastRun = (olimit-op) - 1;
+ assert(olimit >= op);
+ lastRun = (size_t)(olimit-op) - 1;
lastRun -= (lastRun+240)/255;
+ } else {
+ assert(outputDirective == limitedOutput);
+ return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */
}
- if (outputLimited == limitedOutput)
- return 0;
}
if (lastRun >= RUN_MASK) {
size_t accumulator = lastRun - RUN_MASK;
@@ -999,31 +1114,33 @@ _last_literals:
op += lastRun;
}
- if (outputLimited == fillOutput) {
+ if (outputDirective == fillOutput) {
*inputConsumed = (int) (((const char*)ip)-source);
}
DEBUGLOG(5, "LZ4_compress_generic: compressed %i bytes into %i bytes", inputSize, (int)(((char*)op) - dest));
- return (int)(((char*)op) - dest);
+ result = (int)(((char*)op) - dest);
+ assert(result > 0);
+ return result;
}
int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
{
- LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse;
+ LZ4_stream_t_internal* const ctx = & LZ4_initStream(state, sizeof(LZ4_stream_t)) -> internal_donotuse;
+ assert(ctx != NULL);
if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
- LZ4_resetStream((LZ4_stream_t*)state);
if (maxOutputSize >= LZ4_compressBound(inputSize)) {
if (inputSize < LZ4_64Klimit) {
return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, byU16, noDict, noDictIssue, acceleration);
} else {
- const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > MAX_DISTANCE)) ? byPtr : byU32;
+ const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32;
return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
}
} else {
if (inputSize < LZ4_64Klimit) {;
return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
} else {
- const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > MAX_DISTANCE)) ? byPtr : byU32;
+ const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32;
return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
}
}
@@ -1053,7 +1170,7 @@ int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst
return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
}
} else {
- const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
+ const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32;
LZ4_prepareTable(ctx, srcSize, tableType);
return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
}
@@ -1067,7 +1184,7 @@ int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst
return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
}
} else {
- const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
+ const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32;
LZ4_prepareTable(ctx, srcSize, tableType);
return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
}
@@ -1094,23 +1211,25 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
}
-int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize)
+int LZ4_compress_default(const char* src, char* dst, int srcSize, int maxOutputSize)
{
- return LZ4_compress_fast(source, dest, inputSize, maxOutputSize, 1);
+ return LZ4_compress_fast(src, dst, srcSize, maxOutputSize, 1);
}
/* hidden debug function */
/* strangely enough, gcc generates faster code when this function is uncommented, even if unused */
-int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
+int LZ4_compress_fast_force(const char* src, char* dst, int srcSize, int dstCapacity, int acceleration)
{
LZ4_stream_t ctx;
- LZ4_resetStream(&ctx);
+ LZ4_initStream(&ctx, sizeof(ctx));
- if (inputSize < LZ4_64Klimit)
- return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
- else
- return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, sizeof(void*)==8 ? byU32 : byPtr, noDict, noDictIssue, acceleration);
+ if (srcSize < LZ4_64Klimit) {
+ return LZ4_compress_generic(&ctx.internal_donotuse, src, dst, srcSize, NULL, dstCapacity, limitedOutput, byU16, noDict, noDictIssue, acceleration);
+ } else {
+ tableType_t const addrMode = (sizeof(void*) > 4) ? byU32 : byPtr;
+ return LZ4_compress_generic(&ctx.internal_donotuse, src, dst, srcSize, NULL, dstCapacity, limitedOutput, addrMode, noDict, noDictIssue, acceleration);
+ }
}
@@ -1119,7 +1238,8 @@ int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int m
* _continue() call without resetting it. */
static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize)
{
- LZ4_resetStream(state);
+ void* const s = LZ4_initStream(state, sizeof (*state));
+ assert(s != NULL); (void)s;
if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */
return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1);
@@ -1127,8 +1247,8 @@ static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src,
if (*srcSizePtr < LZ4_64Klimit) {
return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, byU16, noDict, noDictIssue, 1);
} else {
- tableType_t const tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
- return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, tableType, noDict, noDictIssue, 1);
+ tableType_t const addrMode = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32;
+ return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, addrMode, noDict, noDictIssue, 1);
} }
}
@@ -1159,14 +1279,40 @@ int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targe
LZ4_stream_t* LZ4_createStream(void)
{
- LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t));
+ LZ4_stream_t* const lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t));
LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */
DEBUGLOG(4, "LZ4_createStream %p", lz4s);
if (lz4s == NULL) return NULL;
- LZ4_resetStream(lz4s);
+ LZ4_initStream(lz4s, sizeof(*lz4s));
return lz4s;
}
+#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 :
+ it reports an aligment of 8-bytes,
+ while actually aligning LZ4_stream_t on 4 bytes. */
+static size_t LZ4_stream_t_alignment(void)
+{
+ struct { char c; LZ4_stream_t t; } t_a;
+ return sizeof(t_a) - sizeof(t_a.t);
+}
+#endif
+
+LZ4_stream_t* LZ4_initStream (void* buffer, size_t size)
+{
+ DEBUGLOG(5, "LZ4_initStream");
+ if (buffer == NULL) return NULL;
+ if (size < sizeof(LZ4_stream_t)) return NULL;
+#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 :
+ it reports an aligment of 8-bytes,
+ while actually aligning LZ4_stream_t on 4 bytes. */
+ if (((size_t)buffer) & (LZ4_stream_t_alignment() - 1)) return NULL; /* alignment check */
+#endif
+ MEM_INIT(buffer, 0, sizeof(LZ4_stream_t));
+ return (LZ4_stream_t*)buffer;
+}
+
+/* resetStream is now deprecated,
+ * prefer initStream() which is more general */
void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
{
DEBUGLOG(5, "LZ4_resetStream (ctx:%p)", LZ4_stream);
@@ -1226,10 +1372,16 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
p+=3;
}
- return dict->dictSize;
+ return (int)dict->dictSize;
}
void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream) {
+ /* Calling LZ4_resetStream_fast() here makes sure that changes will not be
+ * erased by subsequent calls to LZ4_resetStream_fast() in case stream was
+ * marked as having dirty context, e.g. requiring full reset.
+ */
+ LZ4_resetStream_fast(working_stream);
+
if (dictionary_stream != NULL) {
/* If the current offset is zero, we will never look in the
* external dictionary context, since there is no value a table
@@ -1248,7 +1400,8 @@ void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dic
static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, int nextSize)
{
- if (LZ4_dict->currentOffset + nextSize > 0x80000000) { /* potential ptrdiff_t overflow (32-bits mode) */
+ assert(nextSize >= 0);
+ if (LZ4_dict->currentOffset + (unsigned)nextSize > 0x80000000) { /* potential ptrdiff_t overflow (32-bits mode) */
/* rescale hash table */
U32 const delta = LZ4_dict->currentOffset - 64 KB;
const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize;
@@ -1265,7 +1418,10 @@ static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, int nextSize)
}
-int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
+int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream,
+ const char* source, char* dest,
+ int inputSize, int maxOutputSize,
+ int acceleration)
{
const tableType_t tableType = byU32;
LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse;
@@ -1273,12 +1429,12 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i)", inputSize);
- if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */
+ if (streamPtr->dirty) return 0; /* Uninitialized structure detected */
LZ4_renormDictT(streamPtr, inputSize); /* avoid index overflow */
if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
/* invalidate tiny dictionaries */
- if ( (streamPtr->dictSize-1 < 4) /* intentional underflow */
+ if ( (streamPtr->dictSize-1 < 4-1) /* intentional underflow */
&& (dictEnd != (const BYTE*)source) ) {
DEBUGLOG(5, "LZ4_compress_fast_continue: dictSize(%u) at addr:%p is too small", streamPtr->dictSize, streamPtr->dictionary);
streamPtr->dictSize = 0;
@@ -1371,7 +1527,7 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize)
const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize;
if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */
- if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize;
+ if ((U32)dictSize > dict->dictSize) dictSize = (int)dict->dictSize;
memmove(safeBuffer, previousDictEnd - dictSize, dictSize);
@@ -1393,6 +1549,37 @@ typedef enum { decode_full_block = 0, partial_decode = 1 } earlyEnd_directive;
#undef MIN
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
+/* Read the variable-length literal or match length.
+ *
+ * ip - pointer to use as input.
+ * lencheck - end ip. Return an error if ip advances >= lencheck.
+ * loop_check - check ip >= lencheck in body of loop. Returns loop_error if so.
+ * initial_check - check ip >= lencheck before start of loop. Returns initial_error if so.
+ * error (output) - error code. Should be set to 0 before call.
+ */
+typedef enum { loop_error = -2, initial_error = -1, ok = 0 } variable_length_error;
+LZ4_FORCE_INLINE unsigned
+read_variable_length(const BYTE**ip, const BYTE* lencheck, int loop_check, int initial_check, variable_length_error* error)
+{
+ unsigned length = 0;
+ unsigned s;
+ if (initial_check && unlikely((*ip) >= lencheck)) { /* overflow detection */
+ *error = initial_error;
+ return length;
+ }
+ do {
+ s = **ip;
+ (*ip)++;
+ length += s;
+ if (loop_check && unlikely((*ip) >= lencheck)) { /* overflow detection */
+ *error = loop_error;
+ return length;
+ }
+ } while (s==255);
+
+ return length;
+}
+
/*! LZ4_decompress_generic() :
* This generic decompression function covers all use cases.
* It shall be instantiated several times, using different sets of directives.
@@ -1414,234 +1601,360 @@ LZ4_decompress_generic(
const size_t dictSize /* note : = 0 if noDict */
)
{
- const BYTE* ip = (const BYTE*) src;
- const BYTE* const iend = ip + srcSize;
+ if (src == NULL) return -1;
- BYTE* op = (BYTE*) dst;
- BYTE* const oend = op + outputSize;
- BYTE* cpy;
+ { const BYTE* ip = (const BYTE*) src;
+ const BYTE* const iend = ip + srcSize;
- const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize;
- const unsigned inc32table[8] = {0, 1, 2, 1, 0, 4, 4, 4};
- const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3};
+ BYTE* op = (BYTE*) dst;
+ BYTE* const oend = op + outputSize;
+ BYTE* cpy;
- const int safeDecode = (endOnInput==endOnInputSize);
- const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB)));
+ const BYTE* const dictEnd = (dictStart == NULL) ? NULL : dictStart + dictSize;
- /* Set up the "end" pointers for the shortcut. */
- const BYTE* const shortiend = iend - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/;
- const BYTE* const shortoend = oend - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/;
+ const int safeDecode = (endOnInput==endOnInputSize);
+ const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB)));
- DEBUGLOG(5, "LZ4_decompress_generic (srcSize:%i, dstSize:%i)", srcSize, outputSize);
- /* Special cases */
- assert(lowPrefix <= op);
- assert(src != NULL);
- if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */
- if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0 ? 1 : -1);
- if ((endOnInput) && unlikely(srcSize==0)) return -1;
+ /* Set up the "end" pointers for the shortcut. */
+ const BYTE* const shortiend = iend - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/;
+ const BYTE* const shortoend = oend - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/;
- /* Main Loop : decode sequences */
- while (1) {
const BYTE* match;
size_t offset;
+ unsigned token;
+ size_t length;
+
+
+ DEBUGLOG(5, "LZ4_decompress_generic (srcSize:%i, dstSize:%i)", srcSize, outputSize);
+
+ /* Special cases */
+ assert(lowPrefix <= op);
+ if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */
+ if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0 ? 1 : -1);
+ if ((endOnInput) && unlikely(srcSize==0)) return -1;
+
+ /* Currently the fast loop shows a regression on qualcomm arm chips. */
+#if LZ4_FAST_DEC_LOOP
+ if ((oend - op) < FASTLOOP_SAFE_DISTANCE)
+ goto safe_decode;
+
+ /* Fast loop : decode sequences as long as output < iend-FASTLOOP_SAFE_DISTANCE */
+ while (1) {
+ /* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */
+ assert(oend - op >= FASTLOOP_SAFE_DISTANCE);
+
+ token = *ip++;
+ length = token >> ML_BITS; /* literal length */
+
+ assert(!endOnInput || ip <= iend); /* ip < iend before the increment */
+
+ /* decode literal length */
+ if (length == RUN_MASK) {
+ variable_length_error error = ok;
+ length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error);
+ if (error == initial_error) goto _output_error;
+ if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */
+ if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */
+
+ /* copy literals */
+ cpy = op+length;
+ LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH);
+ if ( ((endOnInput) && ((cpy>oend-FASTLOOP_SAFE_DISTANCE) || (ip+length>iend-(2+1+LASTLITERALS))) )
+ || ((!endOnInput) && (cpy>oend-FASTLOOP_SAFE_DISTANCE)) )
+ {
+ goto safe_literal_copy;
+ }
+ LZ4_wildCopy32(op, ip, cpy);
+ ip += length; op = cpy;
+ } else {
+ cpy = op+length;
+ /* We don't need to check oend, since we check it once for each loop below */
+ if ( ((endOnInput) && (ip+16>iend-(2+1+LASTLITERALS))))
+ {
+ goto safe_literal_copy;
+ }
+ /* Literals can only be 14, but hope compilers optimize if we copy by a register size */
+ memcpy(op, ip, 16);
+ ip += length; op = cpy;
+ }
- unsigned const token = *ip++;
- size_t length = token >> ML_BITS; /* literal length */
+ /* get offset */
+ offset = LZ4_readLE16(ip); ip+=2;
+ match = op - offset;
- assert(!endOnInput || ip <= iend); /* ip < iend before the increment */
+ /* get matchlength */
+ length = token & ML_MASK;
- /* A two-stage shortcut for the most common case:
- * 1) If the literal length is 0..14, and there is enough space,
- * enter the shortcut and copy 16 bytes on behalf of the literals
- * (in the fast mode, only 8 bytes can be safely copied this way).
- * 2) Further if the match length is 4..18, copy 18 bytes in a similar
- * manner; but we ensure that there's enough space in the output for
- * those 18 bytes earlier, upon entering the shortcut (in other words,
- * there is a combined check for both stages).
- */
- if ( (endOnInput ? length != RUN_MASK : length <= 8)
- /* strictly "less than" on input, to re-enter the loop with at least one byte */
- && likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)) ) {
- /* Copy the literals */
- memcpy(op, ip, endOnInput ? 16 : 8);
- op += length; ip += length;
-
- /* The second stage: prepare for match copying, decode full info.
- * If it doesn't work out, the info won't be wasted. */
- length = token & ML_MASK; /* match length */
- offset = LZ4_readLE16(ip); ip += 2;
- match = op - offset;
- assert(match <= op); /* check overflow */
-
- /* Do not deal with overlapping matches. */
- if ( (length != ML_MASK)
- && (offset >= 8)
- && (dict==withPrefix64k || match >= lowPrefix) ) {
- /* Copy the match. */
- memcpy(op + 0, match + 0, 8);
- memcpy(op + 8, match + 8, 8);
- memcpy(op +16, match +16, 2);
- op += length + MINMATCH;
- /* Both stages worked, load the next token. */
+ if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */
+
+ if (length == ML_MASK) {
+ variable_length_error error = ok;
+ length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error);
+ if (error != ok) goto _output_error;
+ if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */
+ length += MINMATCH;
+ if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) {
+ goto safe_match_copy;
+ }
+ } else {
+ length += MINMATCH;
+ if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) {
+ goto safe_match_copy;
+ }
+
+ /* Fastpath check: Avoids a branch in LZ4_wildCopy32 if true */
+ if (!(dict == usingExtDict) || (match >= lowPrefix)) {
+ if (offset >= 8) {
+ memcpy(op, match, 8);
+ memcpy(op+8, match+8, 8);
+ memcpy(op+16, match+16, 2);
+ op += length;
+ continue;
+ } } }
+
+ /* match starting within external dictionary */
+ if ((dict==usingExtDict) && (match < lowPrefix)) {
+ if (unlikely(op+length > oend-LASTLITERALS)) {
+ if (partialDecoding) length = MIN(length, (size_t)(oend-op));
+ else goto _output_error; /* doesn't respect parsing restriction */
+ }
+
+ if (length <= (size_t)(lowPrefix-match)) {
+ /* match fits entirely within external dictionary : just copy */
+ memmove(op, dictEnd - (lowPrefix-match), length);
+ op += length;
+ } else {
+ /* match stretches into both external dictionary and current block */
+ size_t const copySize = (size_t)(lowPrefix - match);
+ size_t const restSize = length - copySize;
+ memcpy(op, dictEnd - copySize, copySize);
+ op += copySize;
+ if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */
+ BYTE* const endOfMatch = op + restSize;
+ const BYTE* copyFrom = lowPrefix;
+ while (op < endOfMatch) *op++ = *copyFrom++;
+ } else {
+ memcpy(op, lowPrefix, restSize);
+ op += restSize;
+ } }
continue;
}
- /* The second stage didn't work out, but the info is ready.
- * Propel it right to the point of match copying. */
- goto _copy_match;
- }
+ /* copy match within block */
+ cpy = op + length;
- /* decode literal length */
- if (length == RUN_MASK) {
- unsigned s;
- if (unlikely(endOnInput ? ip >= iend-RUN_MASK : 0)) goto _output_error; /* overflow detection */
- do {
- s = *ip++;
- length += s;
- } while ( likely(endOnInput ? ip<iend-RUN_MASK : 1) & (s==255) );
- if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */
- if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */
+ assert((op <= oend) && (oend-op >= 32));
+ if (unlikely(offset<16)) {
+ LZ4_memcpy_using_offset(op, match, cpy, offset);
+ } else {
+ LZ4_wildCopy32(op, match, cpy);
+ }
+
+ op = cpy; /* wildcopy correction */
}
+ safe_decode:
+#endif
- /* copy literals */
- cpy = op+length;
- LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH);
- if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) )
- || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) )
- {
- if (partialDecoding) {
- if (cpy > oend) { cpy = oend; length = oend-op; } /* Partial decoding : stop in the middle of literal segment */
- if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */
- } else {
- if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */
- if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */
+ /* Main Loop : decode remaining sequences where output < FASTLOOP_SAFE_DISTANCE */
+ while (1) {
+ token = *ip++;
+ length = token >> ML_BITS; /* literal length */
+
+ assert(!endOnInput || ip <= iend); /* ip < iend before the increment */
+
+ /* A two-stage shortcut for the most common case:
+ * 1) If the literal length is 0..14, and there is enough space,
+ * enter the shortcut and copy 16 bytes on behalf of the literals
+ * (in the fast mode, only 8 bytes can be safely copied this way).
+ * 2) Further if the match length is 4..18, copy 18 bytes in a similar
+ * manner; but we ensure that there's enough space in the output for
+ * those 18 bytes earlier, upon entering the shortcut (in other words,
+ * there is a combined check for both stages).
+ */
+ if ( (endOnInput ? length != RUN_MASK : length <= 8)
+ /* strictly "less than" on input, to re-enter the loop with at least one byte */
+ && likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)) ) {
+ /* Copy the literals */
+ memcpy(op, ip, endOnInput ? 16 : 8);
+ op += length; ip += length;
+
+ /* The second stage: prepare for match copying, decode full info.
+ * If it doesn't work out, the info won't be wasted. */
+ length = token & ML_MASK; /* match length */
+ offset = LZ4_readLE16(ip); ip += 2;
+ match = op - offset;
+ assert(match <= op); /* check overflow */
+
+ /* Do not deal with overlapping matches. */
+ if ( (length != ML_MASK)
+ && (offset >= 8)
+ && (dict==withPrefix64k || match >= lowPrefix) ) {
+ /* Copy the match. */
+ memcpy(op + 0, match + 0, 8);
+ memcpy(op + 8, match + 8, 8);
+ memcpy(op +16, match +16, 2);
+ op += length + MINMATCH;
+ /* Both stages worked, load the next token. */
+ continue;
+ }
+
+ /* The second stage didn't work out, but the info is ready.
+ * Propel it right to the point of match copying. */
+ goto _copy_match;
}
- memcpy(op, ip, length);
- ip += length;
- op += length;
- if (!partialDecoding || (cpy == oend)) {
- /* Necessarily EOF, due to parsing restrictions */
- break;
+
+ /* decode literal length */
+ if (length == RUN_MASK) {
+ variable_length_error error = ok;
+ length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error);
+ if (error == initial_error) goto _output_error;
+ if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */
+ if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */
}
- } else {
- LZ4_wildCopy(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */
- ip += length; op = cpy;
- }
+ /* copy literals */
+ cpy = op+length;
+#if LZ4_FAST_DEC_LOOP
+ safe_literal_copy:
+#endif
+ LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH);
+ if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) )
+ || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) )
+ {
+ if (partialDecoding) {
+ if (cpy > oend) { cpy = oend; assert(op<=oend); length = (size_t)(oend-op); } /* Partial decoding : stop in the middle of literal segment */
+ if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */
+ } else {
+ if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */
+ if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */
+ }
+ memcpy(op, ip, length);
+ ip += length;
+ op += length;
+ if (!partialDecoding || (cpy == oend)) {
+ /* Necessarily EOF, due to parsing restrictions */
+ break;
+ }
- /* get offset */
- offset = LZ4_readLE16(ip); ip+=2;
- match = op - offset;
+ } else {
+ LZ4_wildCopy(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */
+ ip += length; op = cpy;
+ }
- /* get matchlength */
- length = token & ML_MASK;
+ /* get offset */
+ offset = LZ4_readLE16(ip); ip+=2;
+ match = op - offset;
-_copy_match:
- if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */
- if (!partialDecoding) {
- assert(oend > op);
- assert(oend - op >= 4);
- LZ4_write32(op, 0); /* silence an msan warning when offset==0; costs <1%; */
- } /* note : when partialDecoding, there is no guarantee that at least 4 bytes remain available in output buffer */
+ /* get matchlength */
+ length = token & ML_MASK;
+
+ _copy_match:
+ if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */
+ if (!partialDecoding) {
+ assert(oend > op);
+ assert(oend - op >= 4);
+ LZ4_write32(op, 0); /* silence an msan warning when offset==0; costs <1%; */
+ } /* note : when partialDecoding, there is no guarantee that at least 4 bytes remain available in output buffer */
+
+ if (length == ML_MASK) {
+ variable_length_error error = ok;
+ length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error);
+ if (error != ok) goto _output_error;
+ if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */
+ }
+ length += MINMATCH;
- if (length == ML_MASK) {
- unsigned s;
- do {
- s = *ip++;
- if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error;
- length += s;
- } while (s==255);
- if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */
- }
- length += MINMATCH;
+#if LZ4_FAST_DEC_LOOP
+ safe_match_copy:
+#endif
+ /* match starting within external dictionary */
+ if ((dict==usingExtDict) && (match < lowPrefix)) {
+ if (unlikely(op+length > oend-LASTLITERALS)) {
+ if (partialDecoding) length = MIN(length, (size_t)(oend-op));
+ else goto _output_error; /* doesn't respect parsing restriction */
+ }
- /* match starting within external dictionary */
- if ((dict==usingExtDict) && (match < lowPrefix)) {
- if (unlikely(op+length > oend-LASTLITERALS)) {
- if (partialDecoding) length = MIN(length, (size_t)(oend-op));
- else goto _output_error; /* doesn't respect parsing restriction */
+ if (length <= (size_t)(lowPrefix-match)) {
+ /* match fits entirely within external dictionary : just copy */
+ memmove(op, dictEnd - (lowPrefix-match), length);
+ op += length;
+ } else {
+ /* match stretches into both external dictionary and current block */
+ size_t const copySize = (size_t)(lowPrefix - match);
+ size_t const restSize = length - copySize;
+ memcpy(op, dictEnd - copySize, copySize);
+ op += copySize;
+ if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */
+ BYTE* const endOfMatch = op + restSize;
+ const BYTE* copyFrom = lowPrefix;
+ while (op < endOfMatch) *op++ = *copyFrom++;
+ } else {
+ memcpy(op, lowPrefix, restSize);
+ op += restSize;
+ } }
+ continue;
}
- if (length <= (size_t)(lowPrefix-match)) {
- /* match fits entirely within external dictionary : just copy */
- memmove(op, dictEnd - (lowPrefix-match), length);
- op += length;
- } else {
- /* match stretches into both external dictionary and current block */
- size_t const copySize = (size_t)(lowPrefix - match);
- size_t const restSize = length - copySize;
- memcpy(op, dictEnd - copySize, copySize);
- op += copySize;
- if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */
- BYTE* const endOfMatch = op + restSize;
- const BYTE* copyFrom = lowPrefix;
- while (op < endOfMatch) *op++ = *copyFrom++;
+ /* copy match within block */
+ cpy = op + length;
+
+ /* partialDecoding : may end anywhere within the block */
+ assert(op<=oend);
+ if (partialDecoding && (cpy > oend-MATCH_SAFEGUARD_DISTANCE)) {
+ size_t const mlen = MIN(length, (size_t)(oend-op));
+ const BYTE* const matchEnd = match + mlen;
+ BYTE* const copyEnd = op + mlen;
+ if (matchEnd > op) { /* overlap copy */
+ while (op < copyEnd) *op++ = *match++;
} else {
- memcpy(op, lowPrefix, restSize);
- op += restSize;
- } }
- continue;
- }
+ memcpy(op, match, mlen);
+ }
+ op = copyEnd;
+ if (op==oend) break;
+ continue;
+ }
- /* copy match within block */
- cpy = op + length;
-
- /* partialDecoding : may not respect endBlock parsing restrictions */
- assert(op<=oend);
- if (partialDecoding && (cpy > oend-MATCH_SAFEGUARD_DISTANCE)) {
- size_t const mlen = MIN(length, (size_t)(oend-op));
- const BYTE* const matchEnd = match + mlen;
- BYTE* const copyEnd = op + mlen;
- if (matchEnd > op) { /* overlap copy */
- while (op < copyEnd) *op++ = *match++;
+ if (unlikely(offset<8)) {
+ op[0] = match[0];
+ op[1] = match[1];
+ op[2] = match[2];
+ op[3] = match[3];
+ match += inc32table[offset];
+ memcpy(op+4, match, 4);
+ match -= dec64table[offset];
} else {
- memcpy(op, match, mlen);
+ memcpy(op, match, 8);
+ match += 8;
}
- op = copyEnd;
- if (op==oend) break;
- continue;
- }
-
- if (unlikely(offset<8)) {
- op[0] = match[0];
- op[1] = match[1];
- op[2] = match[2];
- op[3] = match[3];
- match += inc32table[offset];
- memcpy(op+4, match, 4);
- match -= dec64table[offset];
- } else {
- memcpy(op, match, 8);
- match += 8;
- }
- op += 8;
-
- if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) {
- BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1);
- if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */
- if (op < oCopyLimit) {
- LZ4_wildCopy(op, match, oCopyLimit);
- match += oCopyLimit - op;
- op = oCopyLimit;
+ op += 8;
+
+ if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) {
+ BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1);
+ if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */
+ if (op < oCopyLimit) {
+ LZ4_wildCopy(op, match, oCopyLimit);
+ match += oCopyLimit - op;
+ op = oCopyLimit;
+ }
+ while (op < cpy) *op++ = *match++;
+ } else {
+ memcpy(op, match, 8);
+ if (length > 16) LZ4_wildCopy(op+8, match+8, cpy);
}
- while (op < cpy) *op++ = *match++;
- } else {
- memcpy(op, match, 8);
- if (length > 16) LZ4_wildCopy(op+8, match+8, cpy);
+ op = cpy; /* wildcopy correction */
}
- op = cpy; /* wildcopy correction */
- }
- /* end of decoding */
- if (endOnInput)
- return (int) (((char*)op)-dst); /* Nb of output bytes decoded */
- else
- return (int) (((const char*)ip)-src); /* Nb of input bytes read */
+ /* end of decoding */
+ if (endOnInput)
+ return (int) (((char*)op)-dst); /* Nb of output bytes decoded */
+ else
+ return (int) (((const char*)ip)-src); /* Nb of input bytes read */
- /* Overflow error detected */
-_output_error:
- return (int) (-(((const char*)ip)-src))-1;
+ /* Overflow error detected */
+ _output_error:
+ return (int) (-(((const char*)ip)-src))-1;
+ }
}
@@ -1745,12 +2058,13 @@ int LZ4_decompress_fast_doubleDict(const char* source, char* dest, int originalS
LZ4_streamDecode_t* LZ4_createStreamDecode(void)
{
LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOC_AND_ZERO(sizeof(LZ4_streamDecode_t));
+ LZ4_STATIC_ASSERT(LZ4_STREAMDECODESIZE >= sizeof(LZ4_streamDecode_t_internal)); /* A compilation error here means LZ4_STREAMDECODESIZE is not large enough */
return lz4s;
}
int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream)
{
- if (!LZ4_stream) return 0; /* support free on NULL */
+ if (LZ4_stream == NULL) return 0; /* support free on NULL */
FREEMEM(LZ4_stream);
return 0;
}
@@ -1808,7 +2122,7 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch
assert(lz4sd->extDictSize == 0);
result = LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize);
if (result <= 0) return result;
- lz4sd->prefixSize = result;
+ lz4sd->prefixSize = (size_t)result;
lz4sd->prefixEnd = (BYTE*)dest + result;
} else if (lz4sd->prefixEnd == (BYTE*)dest) {
/* They're rolling the current segment. */
@@ -1821,7 +2135,7 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch
result = LZ4_decompress_safe_doubleDict(source, dest, compressedSize, maxOutputSize,
lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize);
if (result <= 0) return result;
- lz4sd->prefixSize += result;
+ lz4sd->prefixSize += (size_t)result;
lz4sd->prefixEnd += result;
} else {
/* The buffer wraps around, or they're switching to another buffer. */
@@ -1830,7 +2144,7 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch
result = LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize,
lz4sd->externalDict, lz4sd->extDictSize);
if (result <= 0) return result;
- lz4sd->prefixSize = result;
+ lz4sd->prefixSize = (size_t)result;
lz4sd->prefixEnd = (BYTE*)dest + result;
}
@@ -1842,12 +2156,13 @@ int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch
{
LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse;
int result;
+ assert(originalSize >= 0);
if (lz4sd->prefixSize == 0) {
assert(lz4sd->extDictSize == 0);
result = LZ4_decompress_fast(source, dest, originalSize);
if (result <= 0) return result;
- lz4sd->prefixSize = originalSize;
+ lz4sd->prefixSize = (size_t)originalSize;
lz4sd->prefixEnd = (BYTE*)dest + originalSize;
} else if (lz4sd->prefixEnd == (BYTE*)dest) {
if (lz4sd->prefixSize >= 64 KB - 1 || lz4sd->extDictSize == 0)
@@ -1856,7 +2171,7 @@ int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch
result = LZ4_decompress_fast_doubleDict(source, dest, originalSize,
lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize);
if (result <= 0) return result;
- lz4sd->prefixSize += originalSize;
+ lz4sd->prefixSize += (size_t)originalSize;
lz4sd->prefixEnd += originalSize;
} else {
lz4sd->extDictSize = lz4sd->prefixSize;
@@ -1864,7 +2179,7 @@ int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch
result = LZ4_decompress_fast_extDict(source, dest, originalSize,
lz4sd->externalDict, lz4sd->extDictSize);
if (result <= 0) return result;
- lz4sd->prefixSize = originalSize;
+ lz4sd->prefixSize = (size_t)originalSize;
lz4sd->prefixEnd = (BYTE*)dest + originalSize;
}
diff --git a/lib/lz4.h b/lib/lz4.h
index 059ef7c..1589be9 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -51,19 +51,23 @@ extern "C" {
multiple GB/s per core, typically reaching RAM speed limits on multi-core systems.
The LZ4 compression library provides in-memory compression and decompression functions.
+ It gives full buffer control to user.
Compression can be done in:
- a single step (described as Simple Functions)
- a single step, reusing a context (described in Advanced Functions)
- unbounded multiple steps (described as Streaming compression)
- lz4.h provides block compression functions. It gives full buffer control to user.
- Decompressing an lz4-compressed block also requires metadata (such as compressed size).
- Each application is free to encode such metadata in whichever way it wants.
+ lz4.h generates and decodes LZ4-compressed blocks (doc/lz4_Block_format.md).
+ Decompressing a block requires additional metadata, such as its compressed size.
+ Each application is free to encode and pass such metadata in whichever way it wants.
- An additional format, called LZ4 frame specification (doc/lz4_Frame_format.md),
- take care of encoding standard metadata alongside LZ4-compressed blocks.
- Frame format is required for interoperability.
- It is delivered through a companion API, declared in lz4frame.h.
+ lz4.h only handle blocks, it can not generate Frames.
+
+ Blocks are different from Frames (doc/lz4_Frame_format.md).
+ Frames bundle both blocks and metadata in a specified manner.
+ This are required for compressed data to be self-contained and portable.
+ Frame format is delivered through a companion API, declared in lz4frame.h.
+ Note that the `lz4` CLI can only manage frames.
*/
/*^***************************************************************
@@ -92,8 +96,8 @@ extern "C" {
/*------ Version ------*/
#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */
-#define LZ4_VERSION_MINOR 8 /* for new (non-breaking) interface capabilities */
-#define LZ4_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */
+#define LZ4_VERSION_MINOR 9 /* for new (non-breaking) interface capabilities */
+#define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */
#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
@@ -103,7 +107,7 @@ extern "C" {
#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION)
LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; useful to check dll version */
-LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; unseful to check dll version */
+LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; useful to check dll version */
/*-************************************
@@ -112,14 +116,15 @@ LZ4LIB_API const char* LZ4_versionString (void); /**< library version string;
/*!
* LZ4_MEMORY_USAGE :
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
- * Increasing memory usage improves compression ratio
- * Reduced memory usage may improve speed, thanks to cache effect
+ * Increasing memory usage improves compression ratio.
+ * Reduced memory usage may improve speed, thanks to better cache locality.
* Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
*/
#ifndef LZ4_MEMORY_USAGE
# define LZ4_MEMORY_USAGE 14
#endif
+
/*-************************************
* Simple Functions
**************************************/
@@ -130,21 +135,22 @@ LZ4LIB_API const char* LZ4_versionString (void); /**< library version string;
It also runs faster, so it's a recommended setting.
If the function cannot compress 'src' into a more limited 'dst' budget,
compression stops *immediately*, and the function result is zero.
- Note : as a consequence, 'dst' content is not valid.
- Note 2 : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer).
+ In which case, 'dst' content is undefined (invalid).
srcSize : max supported value is LZ4_MAX_INPUT_SIZE.
dstCapacity : size of buffer 'dst' (which must be already allocated)
- return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity)
- or 0 if compression fails */
+ @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity)
+ or 0 if compression fails
+ Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer).
+*/
LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity);
/*! LZ4_decompress_safe() :
compressedSize : is the exact complete size of the compressed block.
dstCapacity : is the size of destination buffer, which must be already allocated.
- return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity)
+ @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity)
If destination buffer is not large enough, decoding will stop and output an error code (negative value).
If the source stream is detected malformed, the function will stop decoding and return a negative result.
- This function is protected against malicious data packets.
+ Note : This function is protected against malicious data packets (never writes outside 'dst' buffer, nor read outside 'source' buffer).
*/
LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity);
@@ -155,8 +161,7 @@ LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSi
#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */
#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16)
-/*!
-LZ4_compressBound() :
+/*! LZ4_compressBound() :
Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible)
This function is primarily useful for memory allocation purposes (destination buffer size).
Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example).
@@ -167,8 +172,7 @@ LZ4_compressBound() :
*/
LZ4LIB_API int LZ4_compressBound(int inputSize);
-/*!
-LZ4_compress_fast() :
+/*! LZ4_compress_fast() :
Same as LZ4_compress_default(), but allows selection of "acceleration" factor.
The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.
@@ -178,13 +182,12 @@ LZ4_compress_fast() :
LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
-/*!
-LZ4_compress_fast_extState() :
- Same compression function, just using an externally allocated memory space to store compression state.
- Use LZ4_sizeofState() to know how much memory must be allocated,
- and allocate it on 8-bytes boundaries (using malloc() typically).
- Then, provide this buffer as 'void* state' to compression function.
-*/
+/*! LZ4_compress_fast_extState() :
+ * Same as LZ4_compress_fast(), using an externally allocated memory space for its state.
+ * Use LZ4_sizeofState() to know how much memory must be allocated,
+ * and allocate it on 8-bytes boundaries (using `malloc()` typically).
+ * Then, provide this buffer as `void* state` to compression function.
+ */
LZ4LIB_API int LZ4_sizeofState(void);
LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
@@ -204,27 +207,6 @@ LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* d
LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize);
-/*! LZ4_decompress_fast() : **unsafe!**
- * This function used to be a bit faster than LZ4_decompress_safe(),
- * though situation has changed in recent versions,
- * and now `LZ4_decompress_safe()` can be as fast and sometimes faster than `LZ4_decompress_fast()`.
- * Moreover, LZ4_decompress_fast() is not protected vs malformed input, as it doesn't perform full validation of compressed data.
- * As a consequence, this function is no longer recommended, and may be deprecated in future versions.
- * It's only remaining specificity is that it can decompress data without knowing its compressed size.
- *
- * originalSize : is the uncompressed size to regenerate.
- * `dst` must be already allocated, its size must be >= 'originalSize' bytes.
- * @return : number of bytes read from source buffer (== compressed size).
- * If the source stream is detected malformed, the function stops decoding and returns a negative result.
- * note : This function requires uncompressed originalSize to be known in advance.
- * The function never writes past the output buffer.
- * However, since it doesn't know its 'src' size, it may read past the intended input.
- * Also, because match offsets are not validated during decoding,
- * reads from 'src' may underflow.
- * Use this function in trusted environment **only**.
- */
-LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
-
/*! LZ4_decompress_safe_partial() :
* Decompress an LZ4 compressed block, of size 'srcSize' at position 'src',
* into destination buffer 'dst' of size 'dstCapacity'.
@@ -257,30 +239,49 @@ LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcS
***********************************************/
typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */
-/*! LZ4_createStream() and LZ4_freeStream() :
- * LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure.
- * LZ4_freeStream() releases its memory.
- */
LZ4LIB_API LZ4_stream_t* LZ4_createStream(void);
LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr);
-/*! LZ4_resetStream() :
- * An LZ4_stream_t structure can be allocated once and re-used multiple times.
- * Use this function to start compressing a new stream.
+/*! LZ4_resetStream_fast() : v1.9.0+
+ * Use this to prepare an LZ4_stream_t for a new chain of dependent blocks
+ * (e.g., LZ4_compress_fast_continue()).
+ *
+ * An LZ4_stream_t must be initialized once before usage.
+ * This is automatically done when created by LZ4_createStream().
+ * However, should the LZ4_stream_t be simply declared on stack (for example),
+ * it's necessary to initialize it first, using LZ4_initStream().
+ *
+ * After init, start any new stream with LZ4_resetStream_fast().
+ * A same LZ4_stream_t can be re-used multiple times consecutively
+ * and compress multiple streams,
+ * provided that it starts each new stream with LZ4_resetStream_fast().
+ *
+ * LZ4_resetStream_fast() is much faster than LZ4_initStream(),
+ * but is not compatible with memory regions containing garbage data.
+ *
+ * Note: it's only useful to call LZ4_resetStream_fast()
+ * in the context of streaming compression.
+ * The *extState* functions perform their own resets.
+ * Invoking LZ4_resetStream_fast() before is redundant, and even counterproductive.
*/
-LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr);
+LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
/*! LZ4_loadDict() :
- * Use this function to load a static dictionary into LZ4_stream_t.
- * Any previous data will be forgotten, only 'dictionary' will remain in memory.
+ * Use this function to reference a static dictionary into LZ4_stream_t.
+ * The dictionary must remain available during compression.
+ * LZ4_loadDict() triggers a reset, so any previous data will be forgotten.
+ * The same dictionary will have to be loaded on decompression side for successful decoding.
+ * Dictionary are useful for better compression of small data (KB range).
+ * While LZ4 accept any input as dictionary,
+ * results are generally better when using Zstandard's Dictionary Builder.
* Loading a size of 0 is allowed, and is the same as reset.
- * @return : dictionary size, in bytes (necessarily <= 64 KB)
+ * @return : loaded dictionary size, in bytes (necessarily <= 64 KB)
*/
LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);
/*! LZ4_compress_fast_continue() :
* Compress 'src' content using data from previously compressed blocks, for better compression ratio.
- * 'dst' buffer must be already allocated.
+ * 'dst' buffer must be already allocated.
* If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
*
* @return : size of compressed block
@@ -288,10 +289,10 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in
*
* Note 1 : Each invocation to LZ4_compress_fast_continue() generates a new block.
* Each block has precise boundaries.
+ * Each block must be decompressed separately, calling LZ4_decompress_*() with relevant metadata.
* It's not possible to append blocks together and expect a single invocation of LZ4_decompress_*() to decompress them together.
- * Each block must be decompressed separately, calling LZ4_decompress_*() with associated metadata.
*
- * Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory!
+ * Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory !
*
* Note 3 : When input is structured as a double-buffer, each buffer can have any size, including < 64 KB.
* Make sure that buffers are separated, by at least one byte.
@@ -299,7 +300,7 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in
*
* Note 4 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
*
- * Note 5 : After an error, the stream status is invalid, it can only be reset or freed.
+ * Note 5 : After an error, the stream status is undefined (invalid), it can only be reset or freed.
*/
LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
@@ -335,7 +336,7 @@ LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_str
*/
LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
-/*! LZ4_decoderRingBufferSize() : v1.8.2
+/*! LZ4_decoderRingBufferSize() : v1.8.2+
* Note : in a ring buffer scenario (optional),
* blocks are presumed decompressed next to each other
* up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize),
@@ -347,7 +348,7 @@ LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const
* or 0 if there is an error (invalid maxBlockSize).
*/
LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize);
-#define LZ4_DECODER_RING_BUFFER_SIZE(mbs) (65536 + 14 + (mbs)) /* for static allocation; mbs presumed valid */
+#define LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize) (65536 + 14 + (maxBlockSize)) /* for static allocation; maxBlockSize presumed valid */
/*! LZ4_decompress_*_continue() :
* These decoding functions allow decompression of consecutive blocks in "streaming" mode.
@@ -375,83 +376,67 @@ LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize);
* then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block.
*/
LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity);
-LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize);
/*! LZ4_decompress_*_usingDict() :
* These decoding functions work the same as
* a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue()
* They are stand-alone, and don't need an LZ4_streamDecode_t structure.
- * Dictionary is presumed stable : it must remain accessible and unmodified during next decompression.
+ * Dictionary is presumed stable : it must remain accessible and unmodified during decompression.
+ * Performance tip : Decompression speed can be substantially increased
+ * when dst == dictStart + dictSize.
*/
LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize);
-LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize);
-/*^**********************************************
+/*^*************************************
* !!!!!! STATIC LINKING ONLY !!!!!!
- ***********************************************/
+ ***************************************/
-/*-************************************
- * Unstable declarations
- **************************************
- * Declarations in this section should be considered unstable.
- * Use at your own peril, etc., etc.
- * They may be removed in the future.
- * Their signatures may change.
- **************************************/
+/*-****************************************************************************
+ * Experimental section
+ *
+ * Symbols declared in this section must be considered unstable. Their
+ * signatures or semantics may change, or they may be removed altogether in the
+ * future. They are therefore only safe to depend on when the caller is
+ * statically linked against the library.
+ *
+ * To protect against unsafe usage, not only are the declarations guarded,
+ * the definitions are hidden by default
+ * when building LZ4 as a shared/dynamic library.
+ *
+ * In order to access these declarations,
+ * define LZ4_STATIC_LINKING_ONLY in your application
+ * before including LZ4's headers.
+ *
+ * In order to make their implementations accessible dynamically, you must
+ * define LZ4_PUBLISH_STATIC_FUNCTIONS when building the LZ4 library.
+ ******************************************************************************/
+
+#ifdef LZ4_PUBLISH_STATIC_FUNCTIONS
+#define LZ4LIB_STATIC_API LZ4LIB_API
+#else
+#define LZ4LIB_STATIC_API
+#endif
#ifdef LZ4_STATIC_LINKING_ONLY
-/*! LZ4_resetStream_fast() :
- * Use this, like LZ4_resetStream(), to prepare a context for a new chain of
- * calls to a streaming API (e.g., LZ4_compress_fast_continue()).
- *
- * Note:
- * Using this in advance of a non- streaming-compression function is redundant,
- * and potentially bad for performance, since they all perform their own custom
- * reset internally.
- *
- * Differences from LZ4_resetStream():
- * When an LZ4_stream_t is known to be in a internally coherent state,
- * it can often be prepared for a new compression with almost no work, only
- * sometimes falling back to the full, expensive reset that is always required
- * when the stream is in an indeterminate state (i.e., the reset performed by
- * LZ4_resetStream()).
- *
- * LZ4_streams are guaranteed to be in a valid state when:
- * - returned from LZ4_createStream()
- * - reset by LZ4_resetStream()
- * - memset(stream, 0, sizeof(LZ4_stream_t)), though this is discouraged
- * - the stream was in a valid state and was reset by LZ4_resetStream_fast()
- * - the stream was in a valid state and was then used in any compression call
- * that returned success
- * - the stream was in an indeterminate state and was used in a compression
- * call that fully reset the state (e.g., LZ4_compress_fast_extState()) and
- * that returned success
- *
- * When a stream isn't known to be in a valid state, it is not safe to pass to
- * any fastReset or streaming function. It must first be cleansed by the full
- * LZ4_resetStream().
- */
-LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
/*! LZ4_compress_fast_extState_fastReset() :
* A variant of LZ4_compress_fast_extState().
*
- * Using this variant avoids an expensive initialization step. It is only safe
- * to call if the state buffer is known to be correctly initialized already
- * (see above comment on LZ4_resetStream_fast() for a definition of "correctly
- * initialized"). From a high level, the difference is that this function
- * initializes the provided state with a call to something like
- * LZ4_resetStream_fast() while LZ4_compress_fast_extState() starts with a
- * call to LZ4_resetStream().
+ * Using this variant avoids an expensive initialization step.
+ * It is only safe to call if the state buffer is known to be correctly initialized already
+ * (see above comment on LZ4_resetStream_fast() for a definition of "correctly initialized").
+ * From a high level, the difference is that
+ * this function initializes the provided state with a call to something like LZ4_resetStream_fast()
+ * while LZ4_compress_fast_extState() starts with a call to LZ4_resetStream().
*/
-LZ4LIB_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
+LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
/*! LZ4_attach_dictionary() :
- * This is an experimental API that allows for the efficient use of a
- * static dictionary many times.
+ * This is an experimental API that allows
+ * efficient use of a static dictionary many times.
*
* Rather than re-loading the dictionary buffer into a working context before
* each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a
@@ -462,8 +447,8 @@ LZ4LIB_API int LZ4_compress_fast_extState_fastReset (void* state, const char* sr
* Currently, only streams which have been prepared by LZ4_loadDict() should
* be expected to work.
*
- * Alternatively, the provided dictionary stream pointer may be NULL, in which
- * case any existing dictionary stream is unset.
+ * Alternatively, the provided dictionaryStream may be NULL,
+ * in which case any existing dictionary stream is unset.
*
* If a dictionary is provided, it replaces any pre-existing stream history.
* The dictionary contents are the only history that can be referenced and
@@ -475,17 +460,18 @@ LZ4LIB_API int LZ4_compress_fast_extState_fastReset (void* state, const char* sr
* stream (and source buffer) must remain in-place / accessible / unchanged
* through the completion of the first compression call on the stream.
*/
-LZ4LIB_API void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream);
+LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream);
#endif
-/*-************************************
- * Private definitions
- **************************************
- * Do not use these definitions.
- * They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`.
- * Using these definitions will expose code to API and/or ABI break in future versions of the library.
- **************************************/
+
+/*-************************************************************
+ * PRIVATE DEFINITIONS
+ **************************************************************
+ * Do not use these definitions directly.
+ * They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`.
+ * Accessing members will expose code to API and/or ABI break in future versions of the library.
+ **************************************************************/
#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2)
#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE)
#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */
@@ -497,7 +483,7 @@ typedef struct LZ4_stream_t_internal LZ4_stream_t_internal;
struct LZ4_stream_t_internal {
uint32_t hashTable[LZ4_HASH_SIZE_U32];
uint32_t currentOffset;
- uint16_t initCheck;
+ uint16_t dirty;
uint16_t tableType;
const uint8_t* dictionary;
const LZ4_stream_t_internal* dictCtx;
@@ -517,7 +503,7 @@ typedef struct LZ4_stream_t_internal LZ4_stream_t_internal;
struct LZ4_stream_t_internal {
unsigned int hashTable[LZ4_HASH_SIZE_U32];
unsigned int currentOffset;
- unsigned short initCheck;
+ unsigned short dirty;
unsigned short tableType;
const unsigned char* dictionary;
const LZ4_stream_t_internal* dictCtx;
@@ -526,38 +512,54 @@ struct LZ4_stream_t_internal {
typedef struct {
const unsigned char* externalDict;
- size_t extDictSize;
const unsigned char* prefixEnd;
+ size_t extDictSize;
size_t prefixSize;
} LZ4_streamDecode_t_internal;
#endif
-/*!
- * LZ4_stream_t :
- * information structure to track an LZ4 stream.
- * init this structure before first use.
- * note : only use in association with static linking !
- * this definition is not API/ABI safe,
- * it may change in a future version !
+/*! LZ4_stream_t :
+ * information structure to track an LZ4 stream.
+ * LZ4_stream_t can also be created using LZ4_createStream(), which is recommended.
+ * The structure definition can be convenient for static allocation
+ * (on stack, or as part of larger structure).
+ * Init this structure with LZ4_initStream() before first use.
+ * note : only use this definition in association with static linking !
+ * this definition is not API/ABI safe, and may change in a future version.
*/
-#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4)
+#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4 + ((sizeof(void*)==16) ? 4 : 0) /*AS-400*/ )
#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long))
union LZ4_stream_u {
unsigned long long table[LZ4_STREAMSIZE_U64];
LZ4_stream_t_internal internal_donotuse;
} ; /* previously typedef'd to LZ4_stream_t */
+/*! LZ4_initStream() : v1.9.0+
+ * An LZ4_stream_t structure must be initialized at least once.
+ * This is automatically done when invoking LZ4_createStream(),
+ * but it's not when the structure is simply declared on stack (for example).
+ *
+ * Use LZ4_initStream() to properly initialize a newly declared LZ4_stream_t.
+ * It can also initialize any arbitrary buffer of sufficient size,
+ * and will @return a pointer of proper type upon initialization.
+ *
+ * Note : initialization fails if size and alignment conditions are not respected.
+ * In which case, the function will @return NULL.
+ * Note2: An LZ4_stream_t structure guarantees correct alignment and size.
+ * Note3: Before v1.9.0, use LZ4_resetStream() instead
+ */
+LZ4LIB_API LZ4_stream_t* LZ4_initStream (void* buffer, size_t size);
-/*!
- * LZ4_streamDecode_t :
- * information structure to track an LZ4 stream during decompression.
- * init this structure using LZ4_setStreamDecode (or memset()) before first use
- * note : only use in association with static linking !
- * this definition is not API/ABI safe,
- * and may change in a future version !
+
+/*! LZ4_streamDecode_t :
+ * information structure to track an LZ4 stream during decompression.
+ * init this structure using LZ4_setStreamDecode() before first use.
+ * note : only use in association with static linking !
+ * this definition is not API/ABI safe,
+ * and may change in a future version !
*/
-#define LZ4_STREAMDECODESIZE_U64 4
+#define LZ4_STREAMDECODESIZE_U64 (4 + ((sizeof(void*)==16) ? 2 : 0) /*AS-400*/ )
#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long))
union LZ4_streamDecode_u {
unsigned long long table[LZ4_STREAMDECODESIZE_U64];
@@ -570,11 +572,16 @@ union LZ4_streamDecode_u {
**************************************/
/*! Deprecation warnings
- Should deprecation warnings be a problem,
- it is generally possible to disable them,
- typically with -Wno-deprecated-declarations for gcc
- or _CRT_SECURE_NO_WARNINGS in Visual.
- Otherwise, it's also possible to define LZ4_DISABLE_DEPRECATE_WARNINGS */
+ *
+ * Deprecated functions make the compiler generate a warning when invoked.
+ * This is meant to invite users to update their source code.
+ * Should deprecation warnings be a problem, it is generally possible to disable them,
+ * typically with -Wno-deprecated-declarations for gcc
+ * or _CRT_SECURE_NO_WARNINGS in Visual.
+ *
+ * Another method is to define LZ4_DISABLE_DEPRECATE_WARNINGS
+ * before including the header file.
+ */
#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS
# define LZ4_DEPRECATED(message) /* disable deprecation warnings */
#else
@@ -594,8 +601,8 @@ union LZ4_streamDecode_u {
#endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */
/* Obsolete compression functions */
-LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* source, char* dest, int sourceSize);
-LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize);
+LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* source, char* dest, int sourceSize);
+LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize);
LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
@@ -616,13 +623,54 @@ LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompres
*/
LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer);
LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int LZ4_sizeofStreamState(void);
-LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer);
-LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer (void* state);
+LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer);
+LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer (void* state);
/* Obsolete streaming decoding functions */
LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
+/*! LZ4_decompress_fast() : **unsafe!**
+ * These functions are generally slightly faster than LZ4_decompress_safe(),
+ * though the difference is small (generally ~5%).
+ * However, the real cost is a risk : LZ4_decompress_safe() is protected vs malformed input, while `LZ4_decompress_fast()` is not, making it a security liability.
+ * As a consequence, LZ4_decompress_fast() is strongly discouraged, and deprecated.
+ * These functions will generate a deprecation warning in the future.
+ *
+ * Last LZ4_decompress_fast() specificity is that it can decompress a block without knowing its compressed size.
+ * Note that even that functionality could be achieved in a more secure manner if need be,
+ * though it would require new prototypes, and adaptation of the implementation to this new use case.
+ *
+ * Parameters:
+ * originalSize : is the uncompressed size to regenerate.
+ * `dst` must be already allocated, its size must be >= 'originalSize' bytes.
+ * @return : number of bytes read from source buffer (== compressed size).
+ * The function expects to finish at block's end exactly.
+ * If the source stream is detected malformed, the function stops decoding and returns a negative result.
+ * note : LZ4_decompress_fast*() requires originalSize. Thanks to this information, it never writes past the output buffer.
+ * However, since it doesn't know its 'src' size, it may read an unknown amount of input, past input buffer bounds.
+ * Also, since match offsets are not validated, match reads from 'src' may underflow too.
+ * These issues never happen if input (compressed) data is correct.
+ * But they may happen if input data is invalid (error or intentional tampering).
+ * As a consequence, use these functions in trusted environments with trusted data **only**.
+ */
+
+/* LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe() instead") */
+LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
+/* LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_continue() instead") */
+LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize);
+/* LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_usingDict() instead") */
+LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize);
+
+/*! LZ4_resetStream() :
+ * An LZ4_stream_t structure must be initialized at least once.
+ * This is done with LZ4_initStream(), or LZ4_resetStream().
+ * Consider switching to LZ4_initStream(),
+ * invoking LZ4_resetStream() will trigger deprecation warnings in the future.
+ */
+LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr);
+
+
#endif /* LZ4_H_2983827168210 */
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 08bf0fa..19efd0b 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -1,41 +1,44 @@
/*
-LZ4 auto-framing library
-Copyright (C) 2011-2016, Yann Collet.
-
-BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-* Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-* Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-You can contact the author at :
-- LZ4 homepage : http://www.lz4.org
-- LZ4 source repository : https://github.com/lz4/lz4
-*/
+ * LZ4 auto-framing library
+ * Copyright (C) 2011-2016, Yann Collet.
+ *
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * You can contact the author at :
+ * - LZ4 homepage : http://www.lz4.org
+ * - LZ4 source repository : https://github.com/lz4/lz4
+ */
/* LZ4F is a stand-alone API to create LZ4-compressed Frames
-* in full conformance with specification v1.5.0
-* All related operations, including memory management, are handled by the library.
-* */
+ * in full conformance with specification v1.6.1 .
+ * This library rely upon memory management capabilities (malloc, free)
+ * provided either by <stdlib.h>,
+ * or redirected towards another library of user's choice
+ * (see Memory Routines below).
+ */
/*-************************************
@@ -62,16 +65,27 @@ You can contact the author at :
/*-************************************
* Memory routines
**************************************/
+/*
+ * User may redirect invocations of
+ * malloc(), calloc() and free()
+ * towards another library or solution of their choice
+ * by modifying below section.
+ */
#include <stdlib.h> /* malloc, calloc, free */
-#define ALLOC(s) malloc(s)
-#define ALLOC_AND_ZERO(s) calloc(1,s)
-#define FREEMEM free
+#define ALLOC(s) malloc(s)
+#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */
+# define ALLOC_AND_ZERO(s) calloc(1,(s))
+#endif
+#define FREEMEM(p) free(p)
+
#include <string.h> /* memset, memcpy, memmove */
-#define MEM_INIT memset
+#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */
+# define MEM_INIT memset
+#endif
/*-************************************
-* Includes
+* Library declarations
**************************************/
#define LZ4F_STATIC_LINKING_ONLY
#include "lz4frame.h"
@@ -134,8 +148,8 @@ static U32 LZ4F_readLE32 (const void* src)
{
const BYTE* const srcPtr = (const BYTE*)src;
U32 value32 = srcPtr[0];
- value32 += (srcPtr[1]<<8);
- value32 += (srcPtr[2]<<16);
+ value32 += ((U32)srcPtr[1])<< 8;
+ value32 += ((U32)srcPtr[2])<<16;
value32 += ((U32)srcPtr[3])<<24;
return value32;
}
@@ -180,9 +194,11 @@ static void LZ4F_writeLE64 (void* dst, U64 value64)
/*-************************************
* Constants
**************************************/
-#define KB *(1<<10)
-#define MB *(1<<20)
-#define GB *(1<<30)
+#ifndef LZ4_SRC_INCLUDED /* avoid double definition */
+# define KB *(1<<10)
+# define MB *(1<<20)
+# define GB *(1<<30)
+#endif
#define _1BIT 0x01
#define _2BITS 0x03
@@ -195,9 +211,10 @@ static void LZ4F_writeLE64 (void* dst, U64 value64)
#define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U
#define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB
-static const size_t minFHSize = 7;
+static const size_t minFHSize = LZ4F_HEADER_SIZE_MIN; /* 7 */
static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 19 */
-static const size_t BHSize = 4;
+static const size_t BHSize = 4; /* block header : size, and compress flag */
+static const size_t BFSize = 4; /* block footer : checksum (optional) */
/*-************************************
@@ -258,22 +275,22 @@ unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; }
int LZ4F_compressionLevel_max(void) { return LZ4HC_CLEVEL_MAX; }
-
-/*-************************************
-* Private functions
-**************************************/
-#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
-
-static size_t LZ4F_getBlockSize(unsigned blockSizeID)
+size_t LZ4F_getBlockSize(unsigned blockSizeID)
{
static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB };
if (blockSizeID == 0) blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT;
- blockSizeID -= 4;
- if (blockSizeID > 3) return err0r(LZ4F_ERROR_maxBlockSize_invalid);
+ if (blockSizeID < LZ4F_max64KB || blockSizeID > LZ4F_max4MB)
+ return err0r(LZ4F_ERROR_maxBlockSize_invalid);
+ blockSizeID -= LZ4F_max64KB;
return blockSizes[blockSizeID];
}
+/*-************************************
+* Private functions
+**************************************/
+#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
+
static BYTE LZ4F_headerChecksum (const void* header, size_t length)
{
U32 const xxh = XXH32(header, length, 0);
@@ -323,11 +340,10 @@ static size_t LZ4F_compressBound_internal(size_t srcSize,
size_t const lastBlockSize = flush ? partialBlockSize : 0;
unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0);
- size_t const blockHeaderSize = 4;
- size_t const blockCRCSize = 4 * prefsPtr->frameInfo.blockChecksumFlag;
- size_t const frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4);
+ size_t const blockCRCSize = BFSize * prefsPtr->frameInfo.blockChecksumFlag;
+ size_t const frameEnd = BHSize + (prefsPtr->frameInfo.contentChecksumFlag*BFSize);
- return ((blockHeaderSize + blockCRCSize) * nbBlocks) +
+ return ((BHSize + blockCRCSize) * nbBlocks) +
(blockSize * nbFullBlocks) + lastBlockSize + frameEnd;
}
}
@@ -388,15 +404,18 @@ size_t LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx,
if (LZ4F_isError(headerSize)) return headerSize;
dstPtr += headerSize; /* header size */ }
- { size_t const cSize = LZ4F_compressUpdate(cctx, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options);
+ assert(dstEnd >= dstPtr);
+ { size_t const cSize = LZ4F_compressUpdate(cctx, dstPtr, (size_t)(dstEnd-dstPtr), srcBuffer, srcSize, &options);
if (LZ4F_isError(cSize)) return cSize;
dstPtr += cSize; }
- { size_t const tailSize = LZ4F_compressEnd(cctx, dstPtr, dstEnd-dstPtr, &options); /* flush last block, and generate suffix */
+ assert(dstEnd >= dstPtr);
+ { size_t const tailSize = LZ4F_compressEnd(cctx, dstPtr, (size_t)(dstEnd-dstPtr), &options); /* flush last block, and generate suffix */
if (LZ4F_isError(tailSize)) return tailSize;
dstPtr += tailSize; }
- return (dstPtr - dstStart);
+ assert(dstEnd >= dstStart);
+ return (size_t)(dstPtr - dstStart);
}
@@ -428,7 +447,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
if (preferencesPtr == NULL ||
preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN)
{
- LZ4_resetStream(&lz4ctx);
+ LZ4_initStream(&lz4ctx, sizeof(lz4ctx));
cctxPtr->lz4CtxPtr = &lz4ctx;
cctxPtr->lz4CtxAlloc = 1;
cctxPtr->lz4CtxState = 1;
@@ -598,20 +617,22 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
if (cctxPtr->lz4CtxAlloc < ctxTypeID) {
FREEMEM(cctxPtr->lz4CtxPtr);
if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
- cctxPtr->lz4CtxPtr = (void*)LZ4_createStream();
+ cctxPtr->lz4CtxPtr = LZ4_createStream();
} else {
- cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC();
+ cctxPtr->lz4CtxPtr = LZ4_createStreamHC();
}
- if (cctxPtr->lz4CtxPtr == NULL) return err0r(LZ4F_ERROR_allocation_failed);
+ if (cctxPtr->lz4CtxPtr == NULL)
+ return err0r(LZ4F_ERROR_allocation_failed);
cctxPtr->lz4CtxAlloc = ctxTypeID;
cctxPtr->lz4CtxState = ctxTypeID;
} else if (cctxPtr->lz4CtxState != ctxTypeID) {
/* otherwise, a sufficient buffer is allocated, but we need to
* reset it to the correct context type */
if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
- LZ4_resetStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr);
+ LZ4_initStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr, sizeof (LZ4_stream_t));
} else {
- LZ4_resetStreamHC((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+ LZ4_initStreamHC((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, sizeof(LZ4_streamHC_t));
+ LZ4_setCompressionLevel((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
}
cctxPtr->lz4CtxState = ctxTypeID;
}
@@ -623,8 +644,8 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
cctxPtr->maxBlockSize = LZ4F_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID);
{ size_t const requiredBuffSize = preferencesPtr->autoFlush ?
- (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 64 KB : /* only needs windows size */
- cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 128 KB);
+ ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) ? 64 KB : 0) : /* only needs past data up to window size */
+ cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) ? 128 KB : 0);
if (cctxPtr->maxBufferSize < requiredBuffSize) {
cctxPtr->maxBufferSize = 0;
@@ -635,7 +656,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
} }
cctxPtr->tmpIn = cctxPtr->tmpBuff;
cctxPtr->tmpInSize = 0;
- XXH32_reset(&(cctxPtr->xxh), 0);
+ (void)XXH32_reset(&(cctxPtr->xxh), 0);
/* context init */
cctxPtr->cdict = cdict;
@@ -644,7 +665,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
LZ4F_initStream(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel, LZ4F_blockLinked);
}
if (preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) {
- LZ4_favorDecompressionSpeed((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed);
+ LZ4_favorDecompressionSpeed((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed);
}
/* Magic Number */
@@ -656,7 +677,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
*dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */
+ ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5)
+ ((cctxPtr->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4)
- + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3)
+ + ((unsigned)(cctxPtr->prefs.frameInfo.contentSize > 0) << 3)
+ ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2)
+ (cctxPtr->prefs.frameInfo.dictID > 0) );
/* BD Byte */
@@ -673,11 +694,11 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
dstPtr += 4;
}
/* Header CRC Byte */
- *dstPtr = LZ4F_headerChecksum(headerStart, dstPtr - headerStart);
+ *dstPtr = LZ4F_headerChecksum(headerStart, (size_t)(dstPtr - headerStart));
dstPtr++;
cctxPtr->cStage = 1; /* header written, now request input data block */
- return (dstPtr - dstStart);
+ return (size_t)(dstPtr - dstStart);
}
@@ -686,7 +707,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
* dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes.
* preferencesPtr can be NULL, in which case default parameters are selected.
* @return : number of bytes written into dstBuffer for the header
- * or an error code (can be tested using LZ4F_isError())
+ * or an error code (can be tested using LZ4F_isError())
*/
size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr,
void* dstBuffer, size_t dstCapacity,
@@ -712,27 +733,31 @@ typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize
/*! LZ4F_makeBlock():
- * compress a single block, add header and checksum
- * assumption : dst buffer capacity is >= srcSize */
-static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize,
+ * compress a single block, add header and optional checksum.
+ * assumption : dst buffer capacity is >= BHSize + srcSize + crcSize
+ */
+static size_t LZ4F_makeBlock(void* dst,
+ const void* src, size_t srcSize,
compressFunc_t compress, void* lz4ctx, int level,
- const LZ4F_CDict* cdict, LZ4F_blockChecksum_t crcFlag)
+ const LZ4F_CDict* cdict,
+ LZ4F_blockChecksum_t crcFlag)
{
BYTE* const cSizePtr = (BYTE*)dst;
- U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4),
+ U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize),
(int)(srcSize), (int)(srcSize-1),
level, cdict);
- LZ4F_writeLE32(cSizePtr, cSize);
if (cSize == 0) { /* compression failed */
cSize = (U32)srcSize;
LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG);
- memcpy(cSizePtr+4, src, srcSize);
+ memcpy(cSizePtr+BHSize, src, srcSize);
+ } else {
+ LZ4F_writeLE32(cSizePtr, cSize);
}
if (crcFlag) {
- U32 const crc32 = XXH32(cSizePtr+4, cSize, 0); /* checksum of compressed data */
- LZ4F_writeLE32(cSizePtr+4+cSize, crc32);
+ U32 const crc32 = XXH32(cSizePtr+BHSize, cSize, 0); /* checksum of compressed data */
+ LZ4F_writeLE32(cSizePtr+BHSize+cSize, crc32);
}
- return 4 + cSize + ((U32)crcFlag)*4;
+ return BHSize + cSize + ((U32)crcFlag)*BFSize;
}
@@ -832,9 +857,11 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy);
srcPtr += sizeToCopy;
- dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, blockSize,
+ dstPtr += LZ4F_makeBlock(dstPtr,
+ cctxPtr->tmpIn, blockSize,
compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
- cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
+ cctxPtr->cdict,
+ cctxPtr->prefs.frameInfo.blockChecksumFlag);
if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
cctxPtr->tmpInSize = 0;
@@ -844,18 +871,22 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
while ((size_t)(srcEnd - srcPtr) >= blockSize) {
/* compress full blocks */
lastBlockCompressed = fromSrcBuffer;
- dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, blockSize,
+ dstPtr += LZ4F_makeBlock(dstPtr,
+ srcPtr, blockSize,
compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
- cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
+ cctxPtr->cdict,
+ cctxPtr->prefs.frameInfo.blockChecksumFlag);
srcPtr += blockSize;
}
if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) {
/* compress remaining input < blockSize */
lastBlockCompressed = fromSrcBuffer;
- dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, srcEnd - srcPtr,
+ dstPtr += LZ4F_makeBlock(dstPtr,
+ srcPtr, (size_t)(srcEnd - srcPtr),
compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
- cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
+ cctxPtr->cdict,
+ cctxPtr->prefs.frameInfo.blockChecksumFlag);
srcPtr = srcEnd;
}
@@ -881,28 +912,30 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
/* some input data left, necessarily < blockSize */
if (srcPtr < srcEnd) {
/* fill tmp buffer */
- size_t const sizeToCopy = srcEnd - srcPtr;
+ size_t const sizeToCopy = (size_t)(srcEnd - srcPtr);
memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy);
cctxPtr->tmpInSize = sizeToCopy;
}
if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled)
- XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize);
+ (void)XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize);
cctxPtr->totalInSize += srcSize;
- return dstPtr - dstStart;
+ return (size_t)(dstPtr - dstStart);
}
/*! LZ4F_flush() :
- * Should you need to create compressed data immediately, without waiting for a block to be filled,
- * you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext.
- * The result of the function is the number of bytes written into dstBuffer
- * (it can be zero, this means there was no data left within compressionContext)
+ * When compressed data must be sent immediately, without waiting for a block to be filled,
+ * invoke LZ4_flush(), which will immediately compress any remaining data stored within LZ4F_cctx.
+ * The result of the function is the number of bytes written into dstBuffer.
+ * It can be zero, this means there was no data left within LZ4F_cctx.
* The function outputs an error code if it fails (can be tested using LZ4F_isError())
- * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
+ * LZ4F_compressOptions_t* is optional. NULL is a valid argument.
*/
-size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* compressOptionsPtr)
+size_t LZ4F_flush(LZ4F_cctx* cctxPtr,
+ void* dstBuffer, size_t dstCapacity,
+ const LZ4F_compressOptions_t* compressOptionsPtr)
{
BYTE* const dstStart = (BYTE*)dstBuffer;
BYTE* dstPtr = dstStart;
@@ -910,52 +943,65 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const
if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */
if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
- if (dstCapacity < (cctxPtr->tmpInSize + 4)) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); /* +4 : block header(4) */
+ if (dstCapacity < (cctxPtr->tmpInSize + BHSize + BFSize))
+ return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
(void)compressOptionsPtr; /* not yet useful */
/* select compression function */
compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
/* compress tmp buffer */
- dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize,
+ dstPtr += LZ4F_makeBlock(dstPtr,
+ cctxPtr->tmpIn, cctxPtr->tmpInSize,
compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
- cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
- if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize;
+ cctxPtr->cdict,
+ cctxPtr->prefs.frameInfo.blockChecksumFlag);
+ assert(((void)"flush overflows dstBuffer!", (size_t)(dstPtr - dstStart) <= dstCapacity));
+
+ if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked)
+ cctxPtr->tmpIn += cctxPtr->tmpInSize;
cctxPtr->tmpInSize = 0;
/* keep tmpIn within limits */
if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) { /* necessarily LZ4F_blockLinked */
- int realDictSize = LZ4F_localSaveDict(cctxPtr);
+ int const realDictSize = LZ4F_localSaveDict(cctxPtr);
cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
}
- return dstPtr - dstStart;
+ return (size_t)(dstPtr - dstStart);
}
/*! LZ4F_compressEnd() :
- * When you want to properly finish the compressed frame, just call LZ4F_compressEnd().
- * It will flush whatever data remained within compressionContext (like LZ4_flush())
- * but also properly finalize the frame, with an endMark and a checksum.
- * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size))
- * The function outputs an error code if it fails (can be tested using LZ4F_isError())
- * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
- * compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same.
+ * When you want to properly finish the compressed frame, just call LZ4F_compressEnd().
+ * It will flush whatever data remained within compressionContext (like LZ4_flush())
+ * but also properly finalize the frame, with an endMark and an (optional) checksum.
+ * LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
+ * @return: the number of bytes written into dstBuffer (necessarily >= 4 (endMark size))
+ * or an error code if it fails (can be tested using LZ4F_isError())
+ * The context can then be used again to compress a new frame, starting with LZ4F_compressBegin().
*/
-size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr)
+size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr,
+ void* dstBuffer, size_t dstCapacity,
+ const LZ4F_compressOptions_t* compressOptionsPtr)
{
BYTE* const dstStart = (BYTE*)dstBuffer;
BYTE* dstPtr = dstStart;
- size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstMaxSize, compressOptionsPtr);
+ size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr);
if (LZ4F_isError(flushSize)) return flushSize;
dstPtr += flushSize;
+ assert(flushSize <= dstCapacity);
+ dstCapacity -= flushSize;
+
+ if (dstCapacity < 4) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
LZ4F_writeLE32(dstPtr, 0);
- dstPtr+=4; /* endMark */
+ dstPtr += 4; /* endMark */
if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) {
U32 const xxh = XXH32_digest(&(cctxPtr->xxh));
+ if (dstCapacity < 8) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
LZ4F_writeLE32(dstPtr, xxh);
dstPtr+=4; /* content Checksum */
}
@@ -968,7 +1014,7 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize,
return err0r(LZ4F_ERROR_frameSize_wrong);
}
- return dstPtr - dstStart;
+ return (size_t)(dstPtr - dstStart);
}
@@ -1049,31 +1095,6 @@ void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx)
}
-/*! LZ4F_headerSize() :
- * @return : size of frame header
- * or an error code, which can be tested using LZ4F_isError()
- */
-static size_t LZ4F_headerSize(const void* src, size_t srcSize)
-{
- /* minimal srcSize to determine header size */
- if (srcSize < 5) return err0r(LZ4F_ERROR_frameHeader_incomplete);
-
- /* special case : skippable frames */
- if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) return 8;
-
- /* control magic number */
- if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER)
- return err0r(LZ4F_ERROR_frameType_unknown);
-
- /* Frame Header Size */
- { BYTE const FLG = ((const BYTE*)src)[4];
- U32 const contentSizeFlag = (FLG>>3) & _1BIT;
- U32 const dictIDFlag = FLG & _1BIT;
- return minFHSize + (contentSizeFlag*8) + (dictIDFlag*4);
- }
-}
-
-
/*! LZ4F_decodeHeader() :
* input : `src` points at the **beginning of the frame**
* output : set internal values of dctx, such as
@@ -1125,7 +1146,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
}
/* Frame Header Size */
- frameHeaderSize = minFHSize + (contentSizeFlag*8) + (dictIDFlag*4);
+ frameHeaderSize = minFHSize + (contentSizeFlag?8:0) + (dictIDFlag?4:0);
if (srcSize < frameHeaderSize) {
/* not enough input to fully decode frame header */
@@ -1146,6 +1167,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
}
/* check header */
+ assert(frameHeaderSize > 5);
{ BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5);
if (HC != srcPtr[frameHeaderSize-1])
return err0r(LZ4F_ERROR_headerChecksum_invalid);
@@ -1169,6 +1191,34 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
}
+/*! LZ4F_headerSize() :
+ * @return : size of frame header
+ * or an error code, which can be tested using LZ4F_isError()
+ */
+size_t LZ4F_headerSize(const void* src, size_t srcSize)
+{
+ if (src == NULL) return err0r(LZ4F_ERROR_srcPtr_wrong);
+
+ /* minimal srcSize to determine header size */
+ if (srcSize < LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH)
+ return err0r(LZ4F_ERROR_frameHeader_incomplete);
+
+ /* special case : skippable frames */
+ if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START)
+ return 8;
+
+ /* control magic number */
+ if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER)
+ return err0r(LZ4F_ERROR_frameType_unknown);
+
+ /* Frame Header Size */
+ { BYTE const FLG = ((const BYTE*)src)[4];
+ U32 const contentSizeFlag = (FLG>>3) & _1BIT;
+ U32 const dictIDFlag = FLG & _1BIT;
+ return minFHSize + (contentSizeFlag?8:0) + (dictIDFlag?4:0);
+ }
+}
+
/*! LZ4F_getFrameInfo() :
* This function extracts frame parameters (max blockSize, frame checksum, etc.).
* Usage is optional. Objective is to provide relevant information for allocation purposes.
@@ -1184,10 +1234,12 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
* note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped.
* note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
*/
-LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoPtr,
- const void* srcBuffer, size_t* srcSizePtr)
+LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
+ LZ4F_frameInfo_t* frameInfoPtr,
+ const void* srcBuffer, size_t* srcSizePtr)
{
- if (dctx->dStage > dstage_storeFrameHeader) { /* assumption : dstage_* header enum at beginning of range */
+ LZ4F_STATIC_ASSERT(dstage_getFrameHeader < dstage_storeFrameHeader);
+ if (dctx->dStage > dstage_storeFrameHeader) {
/* frameInfo already decoded */
size_t o=0, i=0;
*srcSizePtr = 0;
@@ -1200,7 +1252,6 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoP
*srcSizePtr = 0;
return err0r(LZ4F_ERROR_frameDecoding_alreadyStarted);
} else {
- size_t decodeResult;
size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr);
if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; }
if (*srcSizePtr < hSize) {
@@ -1208,16 +1259,16 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoP
return err0r(LZ4F_ERROR_frameHeader_incomplete);
}
- decodeResult = LZ4F_decodeHeader(dctx, srcBuffer, hSize);
- if (LZ4F_isError(decodeResult)) {
- *srcSizePtr = 0;
- } else {
- *srcSizePtr = decodeResult;
- decodeResult = BHSize; /* block header size */
- }
- *frameInfoPtr = dctx->frameInfo;
- return decodeResult;
- } }
+ { size_t decodeResult = LZ4F_decodeHeader(dctx, srcBuffer, hSize);
+ if (LZ4F_isError(decodeResult)) {
+ *srcSizePtr = 0;
+ } else {
+ *srcSizePtr = decodeResult;
+ decodeResult = BHSize; /* block header size */
+ }
+ *frameInfoPtr = dctx->frameInfo;
+ return decodeResult;
+ } } }
}
@@ -1235,9 +1286,10 @@ static void LZ4F_updateDict(LZ4F_dctx* dctx,
return;
}
- if (dstPtr - dstBufferStart + dstSize >= 64 KB) { /* history in dstBuffer becomes large enough to become dictionary */
+ assert(dstPtr >= dstBufferStart);
+ if ((size_t)(dstPtr - dstBufferStart) + dstSize >= 64 KB) { /* history in dstBuffer becomes large enough to become dictionary */
dctx->dict = (const BYTE*)dstBufferStart;
- dctx->dictSize = dstPtr - dstBufferStart + dstSize;
+ dctx->dictSize = (size_t)(dstPtr - dstBufferStart) + dstSize;
return;
}
@@ -1253,7 +1305,7 @@ static void LZ4F_updateDict(LZ4F_dctx* dctx,
}
if (withinTmp) { /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */
- size_t const preserveSize = dctx->tmpOut - dctx->tmpOutBuffer;
+ size_t const preserveSize = (size_t)(dctx->tmpOut - dctx->tmpOutBuffer);
size_t copySize = 64 KB - dctx->tmpOutSize;
const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart;
if (dctx->tmpOutSize > 64 KB) copySize = 0;
@@ -1338,7 +1390,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
case dstage_getFrameHeader:
if ((size_t)(srcEnd-srcPtr) >= maxFHSize) { /* enough to decode - shortcut */
- size_t const hSize = LZ4F_decodeHeader(dctx, srcPtr, srcEnd-srcPtr); /* will update dStage appropriately */
+ size_t const hSize = LZ4F_decodeHeader(dctx, srcPtr, (size_t)(srcEnd-srcPtr)); /* will update dStage appropriately */
if (LZ4F_isError(hSize)) return hSize;
srcPtr += hSize;
break;
@@ -1366,14 +1418,14 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
break;
case dstage_init:
- if (dctx->frameInfo.contentChecksumFlag) XXH32_reset(&(dctx->xxh), 0);
+ if (dctx->frameInfo.contentChecksumFlag) (void)XXH32_reset(&(dctx->xxh), 0);
/* internal buffers allocation */
{ size_t const bufferNeeded = dctx->maxBlockSize
- + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB);
+ + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) ? 128 KB : 0);
if (bufferNeeded > dctx->maxBufferSize) { /* tmp buffers too small */
dctx->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/
FREEMEM(dctx->tmpIn);
- dctx->tmpIn = (BYTE*)ALLOC(dctx->maxBlockSize + 4 /* block checksum */);
+ dctx->tmpIn = (BYTE*)ALLOC(dctx->maxBlockSize + BFSize /* block checksum */);
if (dctx->tmpIn == NULL)
return err0r(LZ4F_ERROR_allocation_failed);
FREEMEM(dctx->tmpOutBuffer);
@@ -1420,7 +1472,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
/* decode block header */
{ size_t const nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU;
- size_t const crcSize = dctx->frameInfo.blockChecksumFlag * 4;
+ size_t const crcSize = dctx->frameInfo.blockChecksumFlag * BFSize;
if (nextCBlockSize==0) { /* frameEnd signal, no more block */
dctx->dStage = dstage_getSuffix;
break;
@@ -1431,7 +1483,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
/* next block is uncompressed */
dctx->tmpInTarget = nextCBlockSize;
if (dctx->frameInfo.blockChecksumFlag) {
- XXH32_reset(&dctx->blockChecksum, 0);
+ (void)XXH32_reset(&dctx->blockChecksum, 0);
}
dctx->dStage = dstage_copyDirect;
break;
@@ -1440,7 +1492,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
dctx->tmpInTarget = nextCBlockSize + crcSize;
dctx->dStage = dstage_getCBlock;
if (dstPtr==dstEnd) {
- nextSrcSizeHint = nextCBlockSize + crcSize + BHSize;
+ nextSrcSizeHint = BHSize + nextCBlockSize + crcSize;
doAnotherStage = 0;
}
break;
@@ -1451,10 +1503,10 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
size_t const sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize);
memcpy(dstPtr, srcPtr, sizeToCopy);
if (dctx->frameInfo.blockChecksumFlag) {
- XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy);
+ (void)XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy);
}
if (dctx->frameInfo.contentChecksumFlag)
- XXH32_update(&dctx->xxh, srcPtr, sizeToCopy);
+ (void)XXH32_update(&dctx->xxh, srcPtr, sizeToCopy);
if (dctx->frameInfo.contentSize)
dctx->frameRemainingSize -= sizeToCopy;
@@ -1474,7 +1526,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
}
dctx->tmpInTarget -= sizeToCopy; /* need to copy more */
nextSrcSizeHint = dctx->tmpInTarget +
- + dctx->frameInfo.contentChecksumFlag * 4 /* block checksum */
+ +(dctx->frameInfo.blockChecksumFlag ? BFSize : 0)
+ BHSize /* next header size */;
doAnotherStage = 0;
break;
@@ -1525,8 +1577,10 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
dctx->tmpInSize += sizeToCopy;
srcPtr += sizeToCopy;
if (dctx->tmpInSize < dctx->tmpInTarget) { /* need more input */
- nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize;
- doAnotherStage=0;
+ nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize)
+ + (dctx->frameInfo.blockChecksumFlag ? BFSize : 0)
+ + BHSize /* next header size */;
+ doAnotherStage = 0;
break;
}
selectedIn = dctx->tmpIn;
@@ -1558,13 +1612,13 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
dict, (int)dictSize);
if (decodedSize < 0) return err0r(LZ4F_ERROR_GENERIC); /* decompression failed */
if (dctx->frameInfo.contentChecksumFlag)
- XXH32_update(&(dctx->xxh), dstPtr, decodedSize);
+ XXH32_update(&(dctx->xxh), dstPtr, (size_t)decodedSize);
if (dctx->frameInfo.contentSize)
- dctx->frameRemainingSize -= decodedSize;
+ dctx->frameRemainingSize -= (size_t)decodedSize;
/* dictionary management */
if (dctx->frameInfo.blockMode==LZ4F_blockLinked)
- LZ4F_updateDict(dctx, dstPtr, decodedSize, dstStart, 0);
+ LZ4F_updateDict(dctx, dstPtr, (size_t)decodedSize, dstStart, 0);
dstPtr += decodedSize;
dctx->dStage = dstage_getBlockHeader;
@@ -1601,10 +1655,10 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
if (decodedSize < 0) /* decompression failed */
return err0r(LZ4F_ERROR_decompressionFailed);
if (dctx->frameInfo.contentChecksumFlag)
- XXH32_update(&(dctx->xxh), dctx->tmpOut, decodedSize);
+ XXH32_update(&(dctx->xxh), dctx->tmpOut, (size_t)decodedSize);
if (dctx->frameInfo.contentSize)
- dctx->frameRemainingSize -= decodedSize;
- dctx->tmpOutSize = decodedSize;
+ dctx->frameRemainingSize -= (size_t)decodedSize;
+ dctx->tmpOutSize = (size_t)decodedSize;
dctx->tmpOutStart = 0;
dctx->dStage = dstage_flushOut;
}
@@ -1732,7 +1786,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
&& ((unsigned)(dctx->dStage)-2 < (unsigned)(dstage_getSuffix)-2) ) /* valid stages : [init ... getSuffix[ */
{
if (dctx->dStage == dstage_flushOut) {
- size_t const preserveSize = dctx->tmpOut - dctx->tmpOutBuffer;
+ size_t const preserveSize = (size_t)(dctx->tmpOut - dctx->tmpOutBuffer);
size_t copySize = 64 KB - dctx->tmpOutSize;
const BYTE* oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart;
if (dctx->tmpOutSize > 64 KB) copySize = 0;
@@ -1756,8 +1810,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
}
}
- *srcSizePtr = (srcPtr - srcStart);
- *dstSizePtr = (dstPtr - dstStart);
+ *srcSizePtr = (size_t)(srcPtr - srcStart);
+ *dstSizePtr = (size_t)(dstPtr - dstStart);
return nextSrcSizeHint;
}
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 75f1fd9..ca20dc9 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -32,11 +32,14 @@
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
-/* LZ4F is a stand-alone API to create LZ4-compressed frames
- * conformant with specification v1.6.1.
- * It also offers streaming capabilities.
+/* LZ4F is a stand-alone API able to create and decode LZ4 frames
+ * conformant with specification v1.6.1 in doc/lz4_Frame_format.md .
+ * Generated frames are compatible with `lz4` CLI.
+ *
+ * LZ4F also offers streaming capabilities.
+ *
* lz4.h is not required when using lz4frame.h,
- * except to get constant such as LZ4_VERSION_NUMBER.
+ * except to extract common constant such as LZ4_VERSION_NUMBER.
* */
#ifndef LZ4F_H_09782039843
@@ -195,7 +198,7 @@ typedef struct {
* Simple compression function
***********************************/
-LZ4FLIB_API int LZ4F_compressionLevel_max(void);
+LZ4FLIB_API int LZ4F_compressionLevel_max(void); /* v1.8.0+ */
/*! LZ4F_compressFrameBound() :
* Returns the maximum possible compressed size with LZ4F_compressFrame() given srcSize and preferences.
@@ -247,7 +250,9 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
/*---- Compression ----*/
-#define LZ4F_HEADER_SIZE_MAX 19 /* LZ4 Frame header size can vary from 7 to 19 bytes */
+#define LZ4F_HEADER_SIZE_MIN 7 /* LZ4 Frame header size can vary, depending on selected paramaters */
+#define LZ4F_HEADER_SIZE_MAX 19
+
/*! LZ4F_compressBegin() :
* will write the frame header into dstBuffer.
* dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
@@ -260,15 +265,19 @@ LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx,
const LZ4F_preferences_t* prefsPtr);
/*! LZ4F_compressBound() :
- * Provides minimum dstCapacity required to guarantee compression success
- * given a srcSize and preferences, covering worst case scenario.
+ * Provides minimum dstCapacity required to guarantee success of
+ * LZ4F_compressUpdate(), given a srcSize and preferences, for a worst case scenario.
+ * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() instead.
+ * Note that the result is only valid for a single invocation of LZ4F_compressUpdate().
+ * When invoking LZ4F_compressUpdate() multiple times,
+ * if the output buffer is gradually filled up instead of emptied and re-used from its start,
+ * one must check if there is enough remaining capacity before each invocation, using LZ4F_compressBound().
+ * @return is always the same for a srcSize and prefsPtr.
* prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario.
- * Estimation is valid for either LZ4F_compressUpdate(), LZ4F_flush() or LZ4F_compressEnd(),
- * Estimation includes the possibility that internal buffer might already be filled by up to (blockSize-1) bytes.
- * It also includes frame footer (ending + checksum), which would have to be generated by LZ4F_compressEnd().
- * Estimation doesn't include frame header, as it was already generated by LZ4F_compressBegin().
- * Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
- * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
+ * tech details :
+ * @return includes the possibility that internal buffer might already be filled by up to (blockSize-1) bytes.
+ * It also includes frame footer (ending + checksum), since it might be generated by LZ4F_compressEnd().
+ * @return doesn't include frame header, as it was already generated by LZ4F_compressBegin().
*/
LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr);
@@ -295,6 +304,7 @@ LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx,
* `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default.
* @return : nb of bytes written into dstBuffer (can be zero, when there is no data stored within cctx)
* or an error code if it fails (which can be tested using LZ4F_isError())
+ * Note : LZ4F_flush() is guaranteed to be successful when dstCapacity >= LZ4F_compressBound(0, prefsPtr).
*/
LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx,
void* dstBuffer, size_t dstCapacity,
@@ -307,6 +317,7 @@ LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx,
* `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default.
* @return : nb of bytes written into dstBuffer, necessarily >= 4 (endMark),
* or an error code if it fails (which can be tested using LZ4F_isError())
+ * Note : LZ4F_compressEnd() is guaranteed to be successful when dstCapacity >= LZ4F_compressBound(0, prefsPtr).
* A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.
*/
LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx,
@@ -345,23 +356,58 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
* Streaming decompression functions
*************************************/
+#define LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH 5
+
+/*! LZ4F_headerSize() : v1.9.0+
+ * Provide the header size of a frame starting at `src`.
+ * `srcSize` must be >= LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH,
+ * which is enough to decode the header length.
+ * @return : size of frame header
+ * or an error code, which can be tested using LZ4F_isError()
+ * note : Frame header size is variable, but is guaranteed to be
+ * >= LZ4F_HEADER_SIZE_MIN bytes, and <= LZ4F_HEADER_SIZE_MAX bytes.
+ */
+size_t LZ4F_headerSize(const void* src, size_t srcSize);
+
/*! LZ4F_getFrameInfo() :
* This function extracts frame parameters (max blockSize, dictID, etc.).
- * Its usage is optional.
- * Extracted information is typically useful for allocation and dictionary.
- * This function works in 2 situations :
- * - At the beginning of a new frame, in which case
- * it will decode information from `srcBuffer`, starting the decoding process.
- * Input size must be large enough to successfully decode the entire frame header.
- * Frame header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes.
- * It's allowed to provide more input data than this minimum.
- * - After decoding has been started.
- * In which case, no input is read, frame parameters are extracted from dctx.
- * - If decoding has barely started, but not yet extracted information from header,
+ * Its usage is optional: user can call LZ4F_decompress() directly.
+ *
+ * Extracted information will fill an existing LZ4F_frameInfo_t structure.
+ * This can be useful for allocation and dictionary identification purposes.
+ *
+ * LZ4F_getFrameInfo() can work in the following situations :
+ *
+ * 1) At the beginning of a new frame, before any invocation of LZ4F_decompress().
+ * It will decode header from `srcBuffer`,
+ * consuming the header and starting the decoding process.
+ *
+ * Input size must be large enough to contain the full frame header.
+ * Frame header size can be known beforehand by LZ4F_headerSize().
+ * Frame header size is variable, but is guaranteed to be >= LZ4F_HEADER_SIZE_MIN bytes,
+ * and not more than <= LZ4F_HEADER_SIZE_MAX bytes.
+ * Hence, blindly providing LZ4F_HEADER_SIZE_MAX bytes or more will always work.
+ * It's allowed to provide more input data than the header size,
+ * LZ4F_getFrameInfo() will only consume the header.
+ *
+ * If input size is not large enough,
+ * aka if it's smaller than header size,
+ * function will fail and return an error code.
+ *
+ * 2) After decoding has been started,
+ * it's possible to invoke LZ4F_getFrameInfo() anytime
+ * to extract already decoded frame parameters stored within dctx.
+ *
+ * Note that, if decoding has barely started,
+ * and not yet read enough information to decode the header,
* LZ4F_getFrameInfo() will fail.
- * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
- * Decompression must resume from (srcBuffer + *srcSizePtr).
- * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
+ *
+ * The number of bytes consumed from srcBuffer will be updated in *srcSizePtr (necessarily <= original value).
+ * LZ4F_getFrameInfo() only consumes bytes when decoding has not yet started,
+ * and when decoding the header has been successful.
+ * Decompression must then resume from (srcBuffer + *srcSizePtr).
+ *
+ * @return : a hint about how many srcSize bytes LZ4F_decompress() expects for next call,
* or an error code which can be tested using LZ4F_isError().
* note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely.
* note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
@@ -427,15 +473,15 @@ LZ4FLIB_API void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx); /* always su
extern "C" {
#endif
-/* These declarations are not stable and may change in the future. They are
- * therefore only safe to depend on when the caller is statically linked
- * against the library. To access their declarations, define
- * LZ4F_STATIC_LINKING_ONLY.
+/* These declarations are not stable and may change in the future.
+ * They are therefore only safe to depend on
+ * when the caller is statically linked against the library.
+ * To access their declarations, define LZ4F_STATIC_LINKING_ONLY.
*
- * There is a further protection mechanism where these symbols aren't published
- * into shared/dynamic libraries. You can override this behavior and force
- * them to be published by defining LZ4F_PUBLISH_STATIC_FUNCTIONS. Use at
- * your own risk.
+ * By default, these symbols aren't published into shared/dynamic libraries.
+ * You can override this behavior and force them to be published
+ * by defining LZ4F_PUBLISH_STATIC_FUNCTIONS.
+ * Use at your own risk.
*/
#ifdef LZ4F_PUBLISH_STATIC_FUNCTIONS
#define LZ4FLIB_STATIC_API LZ4FLIB_API
@@ -471,19 +517,38 @@ extern "C" {
#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM,
/* enum list is exposed, to handle specific errors */
-typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes;
+typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM)
+ _LZ4F_dummy_error_enum_for_c89_never_used } LZ4F_errorCodes;
LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult);
-
+LZ4FLIB_STATIC_API size_t LZ4F_getBlockSize(unsigned);
/**********************************
* Bulk processing dictionary API
*********************************/
+
+/* A Dictionary is useful for the compression of small messages (KB range).
+ * It dramatically improves compression efficiency.
+ *
+ * LZ4 can ingest any input as dictionary, though only the last 64 KB are useful.
+ * Best results are generally achieved by using Zstandard's Dictionary Builder
+ * to generate a high-quality dictionary from a set of samples.
+ *
+ * Loading a dictionary has a cost, since it involves construction of tables.
+ * The Bulk processing dictionary API makes it possible to share this cost
+ * over an arbitrary number of compression jobs, even concurrently,
+ * markedly improving compression latency for these cases.
+ *
+ * The same dictionary will have to be used on the decompression side
+ * for decoding to be successful.
+ * To help identify the correct dictionary at decoding stage,
+ * the frame header allows optional embedding of a dictID field.
+ */
typedef struct LZ4F_CDict_s LZ4F_CDict;
/*! LZ4_createCDict() :
- * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
+ * When compressing multiple messages / blocks using the same dictionary, it's recommended to load it just once.
* LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
* LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
* `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict */
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index e913ee7..f6ed779 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -61,9 +61,14 @@
# pragma clang diagnostic ignored "-Wunused-function"
#endif
+/*=== Enums ===*/
+typedef enum { noDictCtx, usingDictCtxHc } dictCtx_directive;
+
+
#define LZ4_COMMONDEFS_ONLY
+#ifndef LZ4_SRC_INCLUDED
#include "lz4.c" /* LZ4_count, constants, mem */
-
+#endif
/*=== Constants ===*/
#define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH)
@@ -76,12 +81,11 @@
#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG))
#define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */
#define DELTANEXTU16(table, pos) table[(U16)(pos)] /* faster */
+/* Make fields passed to, and updated by LZ4HC_encodeSequence explicit */
+#define UPDATABLE(ip, op, anchor) &ip, &op, &anchor
static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); }
-/*=== Enums ===*/
-typedef enum { noDictCtx, usingDictCtx } dictCtx_directive;
-
/**************************************
* HC Compression
@@ -92,9 +96,9 @@ static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4)
MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
}
-static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
+static void LZ4HC_init_internal (LZ4HC_CCtx_internal* hc4, const BYTE* start)
{
- uptrval startingOffset = hc4->end - hc4->base;
+ uptrval startingOffset = (uptrval)(hc4->end - hc4->base);
if (startingOffset > 1 GB) {
LZ4HC_clearTables(hc4);
startingOffset = 0;
@@ -121,7 +125,7 @@ LZ4_FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
while (idx < target) {
U32 const h = LZ4HC_hashPtr(base+idx);
size_t delta = idx - hashTable[h];
- if (delta>MAX_DISTANCE) delta = MAX_DISTANCE;
+ if (delta>LZ4_DISTANCE_MAX) delta = LZ4_DISTANCE_MAX;
DELTANEXTU16(chainTable, idx) = (U16)delta;
hashTable[h] = idx;
idx++;
@@ -224,14 +228,13 @@ LZ4HC_InsertAndGetWiderMatch (
const U32 dictLimit = hc4->dictLimit;
const BYTE* const lowPrefixPtr = base + dictLimit;
const U32 ipIndex = (U32)(ip - base);
- const U32 lowestMatchIndex = (hc4->lowLimit + 64 KB > ipIndex) ? hc4->lowLimit : ipIndex - MAX_DISTANCE;
+ const U32 lowestMatchIndex = (hc4->lowLimit + 64 KB > ipIndex) ? hc4->lowLimit : ipIndex - LZ4_DISTANCE_MAX;
const BYTE* const dictBase = hc4->dictBase;
int const lookBackLength = (int)(ip-iLowLimit);
int nbAttempts = maxNbAttempts;
- int matchChainPos = 0;
+ U32 matchChainPos = 0;
U32 const pattern = LZ4_read32(ip);
U32 matchIndex;
- U32 dictMatchIndex;
repeat_state_e repeat = rep_untested;
size_t srcPatternLength = 0;
@@ -256,7 +259,7 @@ LZ4HC_InsertAndGetWiderMatch (
if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - lookBackLength + longest - 1)) {
if (LZ4_read32(matchPtr) == pattern) {
int const back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr) : 0;
- matchLength = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
+ matchLength = MINMATCH + (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
matchLength -= back;
if (matchLength > longest) {
longest = matchLength;
@@ -270,7 +273,7 @@ LZ4HC_InsertAndGetWiderMatch (
int back = 0;
const BYTE* vLimit = ip + (dictLimit - matchIndex);
if (vLimit > iHighLimit) vLimit = iHighLimit;
- matchLength = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
+ matchLength = (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
if ((ip+matchLength == vLimit) && (vLimit < iHighLimit))
matchLength += LZ4_count(ip+matchLength, lowPrefixPtr, iHighLimit);
back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictStart) : 0;
@@ -283,14 +286,14 @@ LZ4HC_InsertAndGetWiderMatch (
if (chainSwap && matchLength==longest) { /* better match => select a better chain */
assert(lookBackLength==0); /* search forward only */
- if (matchIndex + longest <= ipIndex) {
+ if (matchIndex + (U32)longest <= ipIndex) {
U32 distanceToNextMatch = 1;
int pos;
for (pos = 0; pos <= longest - MINMATCH; pos++) {
- U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + pos);
+ U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + (U32)pos);
if (candidateDist > distanceToNextMatch) {
distanceToNextMatch = candidateDist;
- matchChainPos = pos;
+ matchChainPos = (U32)pos;
} }
if (distanceToNextMatch > 1) {
if (distanceToNextMatch > matchIndex) break; /* avoid overflow */
@@ -315,7 +318,7 @@ LZ4HC_InsertAndGetWiderMatch (
const BYTE* const matchPtr = base + matchCandidateIdx;
if (LZ4_read32(matchPtr) == pattern) { /* good candidate */
size_t const forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern);
- const BYTE* const lowestMatchPtr = (lowPrefixPtr + MAX_DISTANCE >= ip) ? lowPrefixPtr : ip - MAX_DISTANCE;
+ const BYTE* const lowestMatchPtr = (lowPrefixPtr + LZ4_DISTANCE_MAX >= ip) ? lowPrefixPtr : ip - LZ4_DISTANCE_MAX;
size_t const backLength = LZ4HC_reverseCountPattern(matchPtr, lowestMatchPtr, pattern);
size_t const currentSegmentLength = backLength + forwardPatternLength;
@@ -328,7 +331,7 @@ LZ4HC_InsertAndGetWiderMatch (
size_t const maxML = MIN(currentSegmentLength, srcPatternLength);
if ((size_t)longest < maxML) {
assert(base + matchIndex < ip);
- if (ip - (base+matchIndex) > MAX_DISTANCE) break;
+ if (ip - (base+matchIndex) > LZ4_DISTANCE_MAX) break;
assert(maxML < 2 GB);
longest = (int)maxML;
*matchpos = base + matchIndex; /* virtual pos, relative to ip, to retrieve offset */
@@ -343,16 +346,18 @@ LZ4HC_InsertAndGetWiderMatch (
} } /* PA optimization */
/* follow current chain */
- matchIndex -= DELTANEXTU16(chainTable, matchIndex+matchChainPos);
+ matchIndex -= DELTANEXTU16(chainTable, matchIndex + matchChainPos);
} /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */
- if (dict == usingDictCtx && nbAttempts && ipIndex - lowestMatchIndex < MAX_DISTANCE) {
- size_t const dictEndOffset = dictCtx->end - dictCtx->base;
+ if ( dict == usingDictCtxHc
+ && nbAttempts
+ && ipIndex - lowestMatchIndex < LZ4_DISTANCE_MAX) {
+ size_t const dictEndOffset = (size_t)(dictCtx->end - dictCtx->base);
+ U32 dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
assert(dictEndOffset <= 1 GB);
- dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
matchIndex = dictMatchIndex + lowestMatchIndex - (U32)dictEndOffset;
- while (ipIndex - matchIndex <= MAX_DISTANCE && nbAttempts--) {
+ while (ipIndex - matchIndex <= LZ4_DISTANCE_MAX && nbAttempts--) {
const BYTE* const matchPtr = dictCtx->base + dictMatchIndex;
if (LZ4_read32(matchPtr) == pattern) {
@@ -360,22 +365,19 @@ LZ4HC_InsertAndGetWiderMatch (
int back = 0;
const BYTE* vLimit = ip + (dictEndOffset - dictMatchIndex);
if (vLimit > iHighLimit) vLimit = iHighLimit;
- mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
+ mlt = (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0;
mlt -= back;
if (mlt > longest) {
longest = mlt;
*matchpos = base + matchIndex + back;
*startpos = ip + back;
- }
- }
+ } }
{ U32 const nextOffset = DELTANEXTU16(dictCtx->chainTable, dictMatchIndex);
dictMatchIndex -= nextOffset;
matchIndex -= nextOffset;
- }
- }
- }
+ } } }
return longest;
}
@@ -395,14 +397,6 @@ int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index tabl
return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis, 0 /*chainSwap*/, dict, favorCompressionRatio);
}
-
-
-typedef enum {
- noLimit = 0,
- limitedOutput = 1,
- limitedDestSize = 2,
-} limitedOutput_directive;
-
/* LZ4HC_encodeSequence() :
* @return : 0 if ok,
* 1 if buffer issue detected */
@@ -437,7 +431,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
/* Encode Literal length */
length = (size_t)(*ip - *anchor);
- if ((limit) && ((*op + (length >> 8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */
+ if ((limit) && ((*op + (length / 255) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */
if (length >= RUN_MASK) {
size_t len = length - RUN_MASK;
*token = (RUN_MASK << ML_BITS);
@@ -452,13 +446,13 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
*op += length;
/* Encode Offset */
- assert( (*ip - match) <= MAX_DISTANCE ); /* note : consider providing offset as a value, rather than as a pointer difference */
+ assert( (*ip - match) <= LZ4_DISTANCE_MAX ); /* note : consider providing offset as a value, rather than as a pointer difference */
LZ4_writeLE16(*op, (U16)(*ip-match)); *op += 2;
/* Encode MatchLength */
assert(matchLength >= MINMATCH);
- length = (size_t)(matchLength - MINMATCH);
- if ((limit) && (*op + (length >> 8) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */
+ length = (size_t)matchLength - MINMATCH;
+ if ((limit) && (*op + (length / 255) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */
if (length >= ML_MASK) {
*token += ML_MASK;
length -= ML_MASK;
@@ -511,12 +505,12 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
/* init */
*srcSizePtr = 0;
- if (limit == limitedDestSize) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */
+ if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */
if (inputSize < LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */
/* Main Loop */
while (ip <= mflimit) {
- ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis, dict);
+ ml = LZ4HC_InsertAndFindBestMatch(ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis, dict);
if (ml<MINMATCH) { ip++; continue; }
/* saved, in case we would skip too much */
@@ -533,7 +527,7 @@ _Search2:
if (ml2 == ml) { /* No better match => encode ML1 */
optr = op;
- if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow;
+ if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow;
continue;
}
@@ -581,10 +575,10 @@ _Search3:
if (start2 < ip+ml) ml = (int)(start2 - ip);
/* Now, encode 2 sequences */
optr = op;
- if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow;
+ if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow;
ip = start2;
optr = op;
- if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) goto _dest_overflow;
+ if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml2, ref2, limit, oend)) goto _dest_overflow;
continue;
}
@@ -603,7 +597,7 @@ _Search3:
}
optr = op;
- if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow;
+ if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow;
ip = start3;
ref = ref3;
ml = ml3;
@@ -641,7 +635,7 @@ _Search3:
}
}
optr = op;
- if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow;
+ if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow;
/* ML2 becomes ML1 */
ip = start2; ref = ref2; ml = ml2;
@@ -658,7 +652,7 @@ _last_literals:
{ size_t lastRunSize = (size_t)(iend - anchor); /* literals */
size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255;
size_t const totalSize = 1 + litLength + lastRunSize;
- if (limit == limitedDestSize) oend += LASTLITERALS; /* restore correct value */
+ if (limit == fillOutput) oend += LASTLITERALS; /* restore correct value */
if (limit && (op + totalSize > oend)) {
if (limit == limitedOutput) return 0; /* Check output limit */
/* adapt lastRunSize to fill 'dest' */
@@ -685,7 +679,7 @@ _last_literals:
return (int) (((char*)op)-dest);
_dest_overflow:
- if (limit == limitedDestSize) {
+ if (limit == fillOutput) {
op = optr; /* restore correct out pointer */
goto _last_literals;
}
@@ -735,56 +729,64 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
{ lz4opt,16384,LZ4_OPT_NUM }, /* 12==LZ4HC_CLEVEL_MAX */
};
- DEBUGLOG(4, "LZ4HC_compress_generic(%p, %p, %d)", ctx, src, *srcSizePtr);
+ DEBUGLOG(4, "LZ4HC_compress_generic(ctx=%p, src=%p, srcSize=%d)", ctx, src, *srcSizePtr);
- if (limit == limitedDestSize && dstCapacity < 1) return 0; /* Impossible to store anything */
- if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size (too large or negative) */
+ if (limit == fillOutput && dstCapacity < 1) return 0; /* Impossible to store anything */
+ if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size (too large or negative) */
ctx->end += *srcSizePtr;
if (cLevel < 1) cLevel = LZ4HC_CLEVEL_DEFAULT; /* note : convention is different from lz4frame, maybe something to review */
cLevel = MIN(LZ4HC_CLEVEL_MAX, cLevel);
{ cParams_t const cParam = clTable[cLevel];
HCfavor_e const favor = ctx->favorDecSpeed ? favorDecompressionSpeed : favorCompressionRatio;
- if (cParam.strat == lz4hc)
- return LZ4HC_compress_hashChain(ctx,
+ int result;
+
+ if (cParam.strat == lz4hc) {
+ result = LZ4HC_compress_hashChain(ctx,
src, dst, srcSizePtr, dstCapacity,
cParam.nbSearches, limit, dict);
- assert(cParam.strat == lz4opt);
- return LZ4HC_compress_optimal(ctx,
- src, dst, srcSizePtr, dstCapacity,
- cParam.nbSearches, cParam.targetLength, limit,
- cLevel == LZ4HC_CLEVEL_MAX, /* ultra mode */
- dict, favor);
+ } else {
+ assert(cParam.strat == lz4opt);
+ result = LZ4HC_compress_optimal(ctx,
+ src, dst, srcSizePtr, dstCapacity,
+ (int)cParam.nbSearches, cParam.targetLength, limit,
+ cLevel == LZ4HC_CLEVEL_MAX, /* ultra mode */
+ dict, favor);
+ }
+ if (result <= 0) ctx->dirty = 1;
+ return result;
}
}
static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock);
-static int LZ4HC_compress_generic_noDictCtx (
- LZ4HC_CCtx_internal* const ctx,
- const char* const src,
- char* const dst,
- int* const srcSizePtr,
- int const dstCapacity,
- int cLevel,
- limitedOutput_directive limit
- )
+static int
+LZ4HC_compress_generic_noDictCtx (
+ LZ4HC_CCtx_internal* const ctx,
+ const char* const src,
+ char* const dst,
+ int* const srcSizePtr,
+ int const dstCapacity,
+ int cLevel,
+ limitedOutput_directive limit
+ )
{
assert(ctx->dictCtx == NULL);
return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, noDictCtx);
}
-static int LZ4HC_compress_generic_dictCtx (
- LZ4HC_CCtx_internal* const ctx,
- const char* const src,
- char* const dst,
- int* const srcSizePtr,
- int const dstCapacity,
- int cLevel,
- limitedOutput_directive limit
- )
+static int
+LZ4HC_compress_generic_dictCtx (
+ LZ4HC_CCtx_internal* const ctx,
+ const char* const src,
+ char* const dst,
+ int* const srcSizePtr,
+ int const dstCapacity,
+ int cLevel,
+ limitedOutput_directive limit
+ )
{
- const size_t position = ctx->end - ctx->base - ctx->lowLimit;
+ const size_t position = (size_t)(ctx->end - ctx->base) - ctx->lowLimit;
assert(ctx->dictCtx != NULL);
if (position >= 64 KB) {
ctx->dictCtx = NULL;
@@ -795,19 +797,20 @@ static int LZ4HC_compress_generic_dictCtx (
ctx->compressionLevel = (short)cLevel;
return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
} else {
- return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx);
+ return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtxHc);
}
}
-static int LZ4HC_compress_generic (
- LZ4HC_CCtx_internal* const ctx,
- const char* const src,
- char* const dst,
- int* const srcSizePtr,
- int const dstCapacity,
- int cLevel,
- limitedOutput_directive limit
- )
+static int
+LZ4HC_compress_generic (
+ LZ4HC_CCtx_internal* const ctx,
+ const char* const src,
+ char* const dst,
+ int* const srcSizePtr,
+ int const dstCapacity,
+ int cLevel,
+ limitedOutput_directive limit
+ )
{
if (ctx->dictCtx == NULL) {
return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
@@ -817,24 +820,41 @@ static int LZ4HC_compress_generic (
}
-int LZ4_sizeofStateHC(void) { return sizeof(LZ4_streamHC_t); }
+int LZ4_sizeofStateHC(void) { return (int)sizeof(LZ4_streamHC_t); }
+#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 :
+ * it reports an aligment of 8-bytes,
+ * while actually aligning LZ4_streamHC_t on 4 bytes. */
+static size_t LZ4_streamHC_t_alignment(void)
+{
+ struct { char c; LZ4_streamHC_t t; } t_a;
+ return sizeof(t_a) - sizeof(t_a.t);
+}
+#endif
+
+/* state is presumed correctly initialized,
+ * in which case its size and alignment have already been validate */
int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
{
LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
+#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 :
+ * it reports an aligment of 8-bytes,
+ * while actually aligning LZ4_streamHC_t on 4 bytes. */
+ assert(((size_t)state & (LZ4_streamHC_t_alignment() - 1)) == 0); /* check alignment */
+#endif
if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */
LZ4_resetStreamHC_fast((LZ4_streamHC_t*)state, compressionLevel);
- LZ4HC_init (ctx, (const BYTE*)src);
+ LZ4HC_init_internal (ctx, (const BYTE*)src);
if (dstCapacity < LZ4_compressBound(srcSize))
return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, limitedOutput);
else
- return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, noLimit);
+ return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, notLimited);
}
int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
{
- if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */
- LZ4_resetStreamHC ((LZ4_streamHC_t*)state, compressionLevel);
+ LZ4_streamHC_t* const ctx = LZ4_initStreamHC(state, sizeof(*ctx));
+ if (ctx==NULL) return 0; /* init failure */
return LZ4_compress_HC_extStateHC_fastReset(state, src, dst, srcSize, dstCapacity, compressionLevel);
}
@@ -853,14 +873,14 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in
return cSize;
}
-/* LZ4_compress_HC_destSize() :
- * only compatible with regular HC parser */
-int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int cLevel)
+/* state is presumed sized correctly (>= sizeof(LZ4_streamHC_t)) */
+int LZ4_compress_HC_destSize(void* state, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int cLevel)
{
- LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse;
- LZ4_resetStreamHC((LZ4_streamHC_t*)LZ4HC_Data, cLevel);
- LZ4HC_init(ctx, (const BYTE*) source);
- return LZ4HC_compress_generic(ctx, source, dest, sourceSizePtr, targetDestSize, cLevel, limitedDestSize);
+ LZ4_streamHC_t* const ctx = LZ4_initStreamHC(state, sizeof(*ctx));
+ if (ctx==NULL) return 0; /* init failure */
+ LZ4HC_init_internal(&ctx->internal_donotuse, (const BYTE*) source);
+ LZ4_setCompressionLevel(ctx, cLevel);
+ return LZ4HC_compress_generic(&ctx->internal_donotuse, source, dest, sourceSizePtr, targetDestSize, cLevel, fillOutput);
}
@@ -869,14 +889,16 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i
* Streaming Functions
**************************************/
/* allocation */
-LZ4_streamHC_t* LZ4_createStreamHC(void) {
+LZ4_streamHC_t* LZ4_createStreamHC(void)
+{
LZ4_streamHC_t* const LZ4_streamHCPtr = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t));
if (LZ4_streamHCPtr==NULL) return NULL;
- LZ4_resetStreamHC(LZ4_streamHCPtr, LZ4HC_CLEVEL_DEFAULT);
+ LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); /* full initialization, malloc'ed buffer can be full of garbage */
return LZ4_streamHCPtr;
}
-int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) {
+int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr)
+{
DEBUGLOG(4, "LZ4_freeStreamHC(%p)", LZ4_streamHCPtr);
if (!LZ4_streamHCPtr) return 0; /* support free on NULL */
free(LZ4_streamHCPtr);
@@ -884,29 +906,53 @@ int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) {
}
-/* initialization */
-void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
+LZ4_streamHC_t* LZ4_initStreamHC (void* buffer, size_t size)
{
- LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
- DEBUGLOG(4, "LZ4_resetStreamHC(%p, %d)", LZ4_streamHCPtr, compressionLevel);
+ LZ4_streamHC_t* const LZ4_streamHCPtr = (LZ4_streamHC_t*)buffer;
+ if (buffer == NULL) return NULL;
+ if (size < sizeof(LZ4_streamHC_t)) return NULL;
+#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 :
+ * it reports an aligment of 8-bytes,
+ * while actually aligning LZ4_streamHC_t on 4 bytes. */
+ if (((size_t)buffer) & (LZ4_streamHC_t_alignment() - 1)) return NULL; /* alignment check */
+#endif
+ /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
+ LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= LZ4_STREAMHCSIZE);
+ DEBUGLOG(4, "LZ4_initStreamHC(%p, %u)", LZ4_streamHCPtr, (unsigned)size);
+ /* end-base will trigger a clearTable on starting compression */
LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
LZ4_streamHCPtr->internal_donotuse.base = NULL;
LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
LZ4_streamHCPtr->internal_donotuse.favorDecSpeed = 0;
+ LZ4_streamHCPtr->internal_donotuse.dirty = 0;
+ LZ4_setCompressionLevel(LZ4_streamHCPtr, LZ4HC_CLEVEL_DEFAULT);
+ return LZ4_streamHCPtr;
+}
+
+/* just a stub */
+void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
+{
+ LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr));
LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
}
void LZ4_resetStreamHC_fast (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
{
DEBUGLOG(4, "LZ4_resetStreamHC_fast(%p, %d)", LZ4_streamHCPtr, compressionLevel);
- LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base;
- LZ4_streamHCPtr->internal_donotuse.base = NULL;
- LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
+ if (LZ4_streamHCPtr->internal_donotuse.dirty) {
+ LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr));
+ } else {
+ /* preserve end - base : can trigger clearTable's threshold */
+ LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base;
+ LZ4_streamHCPtr->internal_donotuse.base = NULL;
+ LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
+ }
LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
}
void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
{
+ DEBUGLOG(5, "LZ4_setCompressionLevel(%p, %d)", LZ4_streamHCPtr, compressionLevel);
if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT;
if (compressionLevel > LZ4HC_CLEVEL_MAX) compressionLevel = LZ4HC_CLEVEL_MAX;
LZ4_streamHCPtr->internal_donotuse.compressionLevel = (short)compressionLevel;
@@ -917,16 +963,24 @@ void LZ4_favorDecompressionSpeed(LZ4_streamHC_t* LZ4_streamHCPtr, int favor)
LZ4_streamHCPtr->internal_donotuse.favorDecSpeed = (favor!=0);
}
-int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize)
+/* LZ4_loadDictHC() :
+ * LZ4_streamHCPtr is presumed properly initialized */
+int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr,
+ const char* dictionary, int dictSize)
{
LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
DEBUGLOG(4, "LZ4_loadDictHC(%p, %p, %d)", LZ4_streamHCPtr, dictionary, dictSize);
+ assert(LZ4_streamHCPtr != NULL);
if (dictSize > 64 KB) {
- dictionary += dictSize - 64 KB;
+ dictionary += (size_t)dictSize - 64 KB;
dictSize = 64 KB;
}
- LZ4_resetStreamHC(LZ4_streamHCPtr, ctxPtr->compressionLevel);
- LZ4HC_init (ctxPtr, (const BYTE*)dictionary);
+ /* need a full initialization, there are bad side-effects when using resetFast() */
+ { int const cLevel = ctxPtr->compressionLevel;
+ LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr));
+ LZ4_setCompressionLevel(LZ4_streamHCPtr, cLevel);
+ }
+ LZ4HC_init_internal (ctxPtr, (const BYTE*)dictionary);
ctxPtr->end = (const BYTE*)dictionary + dictSize;
if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);
return dictSize;
@@ -959,9 +1013,11 @@ static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr,
limitedOutput_directive limit)
{
LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
- DEBUGLOG(4, "LZ4_compressHC_continue_generic(%p, %p, %d)", LZ4_streamHCPtr, src, *srcSizePtr);
+ DEBUGLOG(4, "LZ4_compressHC_continue_generic(ctx=%p, src=%p, srcSize=%d)",
+ LZ4_streamHCPtr, src, *srcSizePtr);
+ assert(ctxPtr != NULL);
/* auto-init if forgotten */
- if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) src);
+ if (ctxPtr->base == NULL) LZ4HC_init_internal (ctxPtr, (const BYTE*) src);
/* Check overflow */
if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) {
@@ -971,7 +1027,8 @@ static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr,
}
/* Check if blocks follow each other */
- if ((const BYTE*)src != ctxPtr->end) LZ4HC_setExternalDict(ctxPtr, (const BYTE*)src);
+ if ((const BYTE*)src != ctxPtr->end)
+ LZ4HC_setExternalDict(ctxPtr, (const BYTE*)src);
/* Check overlapping input/dictionary space */
{ const BYTE* sourceEnd = (const BYTE*) src + *srcSizePtr;
@@ -992,12 +1049,12 @@ int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src,
if (dstCapacity < LZ4_compressBound(srcSize))
return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, limitedOutput);
else
- return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, noLimit);
+ return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, notLimited);
}
int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int* srcSizePtr, int targetDestSize)
{
- return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, dst, srcSizePtr, targetDestSize, limitedDestSize);
+ return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, dst, srcSizePtr, targetDestSize, fillOutput);
}
@@ -1016,19 +1073,21 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS
{ U32 const endIndex = (U32)(streamPtr->end - streamPtr->base);
streamPtr->end = (const BYTE*)safeBuffer + dictSize;
streamPtr->base = streamPtr->end - endIndex;
- streamPtr->dictLimit = endIndex - dictSize;
- streamPtr->lowLimit = endIndex - dictSize;
+ streamPtr->dictLimit = endIndex - (U32)dictSize;
+ streamPtr->lowLimit = endIndex - (U32)dictSize;
if (streamPtr->nextToUpdate < streamPtr->dictLimit) streamPtr->nextToUpdate = streamPtr->dictLimit;
}
return dictSize;
}
-/***********************************
+/***************************************************
* Deprecated Functions
-***********************************/
+***************************************************/
+
/* These functions currently generate deprecation warnings */
-/* Deprecated compression functions */
+
+/* Wrappers for deprecated compression functions */
int LZ4_compressHC(const char* src, char* dst, int srcSize) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), 0); }
int LZ4_compressHC_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, 0); }
int LZ4_compressHC2(const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); }
@@ -1044,25 +1103,26 @@ int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* ctx, const char* src,
/* Deprecated streaming functions */
int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; }
+/* state is presumed correctly sized, aka >= sizeof(LZ4_streamHC_t)
+ * @return : 0 on success, !=0 if error */
int LZ4_resetStreamStateHC(void* state, char* inputBuffer)
{
- LZ4HC_CCtx_internal *ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
- if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */
- LZ4_resetStreamHC((LZ4_streamHC_t*)state, ((LZ4_streamHC_t*)state)->internal_donotuse.compressionLevel);
- LZ4HC_init(ctx, (const BYTE*)inputBuffer);
+ LZ4_streamHC_t* const hc4 = LZ4_initStreamHC(state, sizeof(*hc4));
+ if (hc4 == NULL) return 1; /* init failed */
+ LZ4HC_init_internal (&hc4->internal_donotuse, (const BYTE*)inputBuffer);
return 0;
}
void* LZ4_createHC (const char* inputBuffer)
{
- LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t));
+ LZ4_streamHC_t* const hc4 = LZ4_createStreamHC();
if (hc4 == NULL) return NULL; /* not enough memory */
- LZ4_resetStreamHC(hc4, 0 /* compressionLevel */);
- LZ4HC_init (&hc4->internal_donotuse, (const BYTE*)inputBuffer);
+ LZ4HC_init_internal (&hc4->internal_donotuse, (const BYTE*)inputBuffer);
return hc4;
}
-int LZ4_freeHC (void* LZ4HC_Data) {
+int LZ4_freeHC (void* LZ4HC_Data)
+{
if (!LZ4HC_Data) return 0; /* support free on NULL */
FREEMEM(LZ4HC_Data);
return 0;
@@ -1070,7 +1130,7 @@ int LZ4_freeHC (void* LZ4HC_Data) {
int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int cLevel)
{
- return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, 0, cLevel, noLimit);
+ return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, 0, cLevel, notLimited);
}
int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int dstCapacity, int cLevel)
@@ -1089,7 +1149,7 @@ char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
/* ================================================
- * LZ4 Optimal parser (levels 10-12)
+ * LZ4 Optimal parser (levels [LZ4HC_CLEVEL_OPT_MIN - LZ4HC_CLEVEL_MAX])
* ===============================================*/
typedef struct {
int price;
@@ -1102,8 +1162,9 @@ typedef struct {
LZ4_FORCE_INLINE int LZ4HC_literalsPrice(int const litlen)
{
int price = litlen;
+ assert(litlen >= 0);
if (litlen >= (int)RUN_MASK)
- price += 1 + (litlen-RUN_MASK)/255;
+ price += 1 + ((litlen-(int)RUN_MASK) / 255);
return price;
}
@@ -1112,11 +1173,13 @@ LZ4_FORCE_INLINE int LZ4HC_literalsPrice(int const litlen)
LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen)
{
int price = 1 + 2 ; /* token + 16-bit offset */
+ assert(litlen >= 0);
+ assert(mlen >= MINMATCH);
price += LZ4HC_literalsPrice(litlen);
if (mlen >= (int)(ML_MASK+MINMATCH))
- price += 1 + (mlen-(ML_MASK+MINMATCH))/255;
+ price += 1 + ((mlen-(int)(ML_MASK+MINMATCH)) / 255);
return price;
}
@@ -1175,9 +1238,9 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx,
BYTE* oend = op + dstCapacity;
/* init */
- DEBUGLOG(5, "LZ4HC_compress_optimal");
+ DEBUGLOG(5, "LZ4HC_compress_optimal(dst=%p, dstCapa=%u)", dst, (unsigned)dstCapacity);
*srcSizePtr = 0;
- if (limit == limitedDestSize) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */
+ if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */
if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1;
/* Main Loop */
@@ -1195,7 +1258,7 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx,
int const firstML = firstMatch.len;
const BYTE* const matchPos = ip - firstMatch.off;
opSaved = op;
- if ( LZ4HC_encodeSequence(&ip, &op, &anchor, firstML, matchPos, limit, oend) ) /* updates ip, op and anchor */
+ if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), firstML, matchPos, limit, oend) ) /* updates ip, op and anchor */
goto _dest_overflow;
continue;
}
@@ -1365,9 +1428,9 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx,
if (ml == 1) { ip++; rPos++; continue; } /* literal; note: can end up with several literals, in which case, skip them */
rPos += ml;
assert(ml >= MINMATCH);
- assert((offset >= 1) && (offset <= MAX_DISTANCE));
+ assert((offset >= 1) && (offset <= LZ4_DISTANCE_MAX));
opSaved = op;
- if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) ) /* updates ip, op and anchor */
+ if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ip - offset, limit, oend) ) /* updates ip, op and anchor */
goto _dest_overflow;
} }
} /* while (ip <= mflimit) */
@@ -1377,7 +1440,7 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx,
{ size_t lastRunSize = (size_t)(iend - anchor); /* literals */
size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255;
size_t const totalSize = 1 + litLength + lastRunSize;
- if (limit == limitedDestSize) oend += LASTLITERALS; /* restore correct value */
+ if (limit == fillOutput) oend += LASTLITERALS; /* restore correct value */
if (limit && (op + totalSize > oend)) {
if (limit == limitedOutput) return 0; /* Check output limit */
/* adapt lastRunSize to fill 'dst' */
@@ -1404,7 +1467,7 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx,
return (int) ((char*)op-dst);
_dest_overflow:
- if (limit == limitedDestSize) {
+ if (limit == fillOutput) {
op = opSaved; /* restore correct out pointer */
goto _last_literals;
}
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 970fa39..cdc6d89 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -54,7 +54,7 @@ extern "C" {
* Block Compression
**************************************/
/*! LZ4_compress_HC() :
- * Compress data from `src` into `dst`, using the more powerful but slower "HC" algorithm.
+ * Compress data from `src` into `dst`, using the powerful but slower "HC" algorithm.
* `dst` must be already allocated.
* Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h")
* Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h")
@@ -77,7 +77,21 @@ LZ4LIB_API int LZ4_compress_HC (const char* src, char* dst, int srcSize, int dst
* Memory segment must be aligned on 8-bytes boundaries (which a normal malloc() should do properly).
*/
LZ4LIB_API int LZ4_sizeofStateHC(void);
-LZ4LIB_API int LZ4_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel);
+LZ4LIB_API int LZ4_compress_HC_extStateHC(void* stateHC, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel);
+
+
+/*! LZ4_compress_HC_destSize() : v1.9.0+
+ * Will compress as much data as possible from `src`
+ * to fit into `targetDstSize` budget.
+ * Result is provided in 2 parts :
+ * @return : the number of bytes written into 'dst' (necessarily <= targetDstSize)
+ * or 0 if compression fails.
+ * `srcSizePtr` : on success, *srcSizePtr is updated to indicate how much bytes were read from `src`
+ */
+LZ4LIB_API int LZ4_compress_HC_destSize(void* stateHC,
+ const char* src, char* dst,
+ int* srcSizePtr, int targetDstSize,
+ int compressionLevel);
/*-************************************
@@ -89,46 +103,92 @@ LZ4LIB_API int LZ4_compress_HC_extStateHC(void* state, const char* src, char* ds
/*! LZ4_createStreamHC() and LZ4_freeStreamHC() :
* These functions create and release memory for LZ4 HC streaming state.
* Newly created states are automatically initialized.
- * Existing states can be re-used several times, using LZ4_resetStreamHC().
- * These methods are API and ABI stable, they can be used in combination with a DLL.
+ * A same state can be used multiple times consecutively,
+ * starting with LZ4_resetStreamHC_fast() to start a new stream of blocks.
*/
LZ4LIB_API LZ4_streamHC_t* LZ4_createStreamHC(void);
LZ4LIB_API int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr);
-LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionLevel);
-LZ4LIB_API int LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize);
-
-LZ4LIB_API int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, const char* src, char* dst, int srcSize, int maxDstSize);
-
-LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize);
-
/*
- These functions compress data in successive blocks of any size, using previous blocks as dictionary.
+ These functions compress data in successive blocks of any size,
+ using previous blocks as dictionary, to improve compression ratio.
One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks.
There is an exception for ring buffers, which can be smaller than 64 KB.
- Ring buffers scenario is automatically detected and handled by LZ4_compress_HC_continue().
+ Ring-buffer scenario is automatically detected and handled within LZ4_compress_HC_continue().
+
+ Before starting compression, state must be allocated and properly initialized.
+ LZ4_createStreamHC() does both, though compression level is set to LZ4HC_CLEVEL_DEFAULT.
+
+ Selecting the compression level can be done with LZ4_resetStreamHC_fast() (starts a new stream)
+ or LZ4_setCompressionLevel() (anytime, between blocks in the same stream) (experimental).
+ LZ4_resetStreamHC_fast() only works on states which have been properly initialized at least once,
+ which is automatically the case when state is created using LZ4_createStreamHC().
+
+ After reset, a first "fictional block" can be designated as initial dictionary,
+ using LZ4_loadDictHC() (Optional).
+
+ Invoke LZ4_compress_HC_continue() to compress each successive block.
+ The number of blocks is unlimited.
+ Previous input blocks, including initial dictionary when present,
+ must remain accessible and unmodified during compression.
+
+ It's allowed to update compression level anytime between blocks,
+ using LZ4_setCompressionLevel() (experimental).
+
+ 'dst' buffer should be sized to handle worst case scenarios
+ (see LZ4_compressBound(), it ensures compression success).
+ In case of failure, the API does not guarantee recovery,
+ so the state _must_ be reset.
+ To ensure compression success
+ whenever `dst` buffer size cannot be made >= LZ4_compressBound(),
+ consider using LZ4_compress_HC_continue_destSize().
+
+ Whenever previous input blocks can't be preserved unmodified in-place during compression of next blocks,
+ it's possible to copy the last blocks into a more stable memory space, using LZ4_saveDictHC().
+ Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer' (<= 64 KB)
+
+ After completing a streaming compression,
+ it's possible to start a new stream of blocks, using the same LZ4_streamHC_t state,
+ just by resetting it, using LZ4_resetStreamHC_fast().
+*/
- Before starting compression, state must be properly initialized, using LZ4_resetStreamHC().
- A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional).
+LZ4LIB_API void LZ4_resetStreamHC_fast(LZ4_streamHC_t* streamHCPtr, int compressionLevel); /* v1.9.0+ */
+LZ4LIB_API int LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize);
+
+LZ4LIB_API int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr,
+ const char* src, char* dst,
+ int srcSize, int maxDstSize);
+
+/*! LZ4_compress_HC_continue_destSize() : v1.9.0+
+ * Similar to LZ4_compress_HC_continue(),
+ * but will read as much data as possible from `src`
+ * to fit into `targetDstSize` budget.
+ * Result is provided into 2 parts :
+ * @return : the number of bytes written into 'dst' (necessarily <= targetDstSize)
+ * or 0 if compression fails.
+ * `srcSizePtr` : on success, *srcSizePtr will be updated to indicate how much bytes were read from `src`.
+ * Note that this function may not consume the entire input.
+ */
+LZ4LIB_API int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr,
+ const char* src, char* dst,
+ int* srcSizePtr, int targetDstSize);
+
+LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize);
- Then, use LZ4_compress_HC_continue() to compress each successive block.
- Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression.
- 'dst' buffer should be sized to handle worst case scenarios (see LZ4_compressBound()), to ensure operation success.
- Because in case of failure, the API does not guarantee context recovery, and context will have to be reset.
- If `dst` buffer budget cannot be >= LZ4_compressBound(), consider using LZ4_compress_HC_continue_destSize() instead.
- If, for any reason, previous data block can't be preserved unmodified in memory for next compression block,
- you can save it to a more stable memory space, using LZ4_saveDictHC().
- Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer'.
-*/
+/*^**********************************************
+ * !!!!!! STATIC LINKING ONLY !!!!!!
+ ***********************************************/
-/*-**************************************************************
+/*-******************************************************************
* PRIVATE DEFINITIONS :
- * Do not use these definitions.
- * They are exposed to allow static allocation of `LZ4_streamHC_t`.
- * Using these definitions makes the code vulnerable to potential API break when upgrading LZ4
- ****************************************************************/
+ * Do not use these definitions directly.
+ * They are merely exposed to allow static allocation of `LZ4_streamHC_t`.
+ * Declare an `LZ4_streamHC_t` directly, rather than any type below.
+ * Even then, only do so in the context of static linking, as definitions may change between versions.
+ ********************************************************************/
+
#define LZ4HC_DICTIONARY_LOGSIZE 16
#define LZ4HC_MAXD (1<<LZ4HC_DICTIONARY_LOGSIZE)
#define LZ4HC_MAXD_MASK (LZ4HC_MAXD - 1)
@@ -153,7 +213,9 @@ struct LZ4HC_CCtx_internal
uint32_t lowLimit; /* below that point, no more dict */
uint32_t nextToUpdate; /* index from which to continue dictionary update */
short compressionLevel;
- short favorDecSpeed;
+ int8_t favorDecSpeed; /* favor decompression speed if this flag set,
+ otherwise, favor compression ratio */
+ int8_t dirty; /* stream has to be fully reset if this flag is set */
const LZ4HC_CCtx_internal* dictCtx;
};
@@ -171,26 +233,43 @@ struct LZ4HC_CCtx_internal
unsigned int lowLimit; /* below that point, no more dict */
unsigned int nextToUpdate; /* index from which to continue dictionary update */
short compressionLevel;
- short favorDecSpeed;
+ char favorDecSpeed; /* favor decompression speed if this flag set,
+ otherwise, favor compression ratio */
+ char dirty; /* stream has to be fully reset if this flag is set */
const LZ4HC_CCtx_internal* dictCtx;
};
#endif
-#define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56) /* 262200 */
+
+/* Do not use these definitions directly !
+ * Declare or allocate an LZ4_streamHC_t instead.
+ */
+#define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56 + ((sizeof(void*)==16) ? 56 : 0) /* AS400*/ ) /* 262200 or 262256*/
#define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t))
union LZ4_streamHC_u {
size_t table[LZ4_STREAMHCSIZE_SIZET];
LZ4HC_CCtx_internal internal_donotuse;
-}; /* previously typedef'd to LZ4_streamHC_t */
-/*
- LZ4_streamHC_t :
- This structure allows static allocation of LZ4 HC streaming state.
- State must be initialized using LZ4_resetStreamHC() before first use.
+}; /* previously typedef'd to LZ4_streamHC_t */
- Static allocation shall only be used in combination with static linking.
- When invoking LZ4 from a DLL, use create/free functions instead, which are API and ABI stable.
-*/
+/* LZ4_streamHC_t :
+ * This structure allows static allocation of LZ4 HC streaming state.
+ * This can be used to allocate statically, on state, or as part of a larger structure.
+ *
+ * Such state **must** be initialized using LZ4_initStreamHC() before first use.
+ *
+ * Note that invoking LZ4_initStreamHC() is not required when
+ * the state was created using LZ4_createStreamHC() (which is recommended).
+ * Using the normal builder, a newly created state is automatically initialized.
+ *
+ * Static allocation shall only be used in combination with static linking.
+ */
+
+/* LZ4_initStreamHC() : v1.9.0+
+ * Required before first use of a statically allocated LZ4_streamHC_t.
+ * Before v1.9.0 : use LZ4_resetStreamHC() instead
+ */
+LZ4LIB_API LZ4_streamHC_t* LZ4_initStreamHC (void* buffer, size_t size);
/*-************************************
@@ -201,11 +280,11 @@ union LZ4_streamHC_u {
/* deprecated compression functions */
LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC (const char* source, char* dest, int inputSize);
LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
-LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel);
-LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel);
+LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize);
LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
-LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel);
+LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel);
LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
@@ -221,10 +300,21 @@ LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_comp
LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (const char* inputBuffer);
LZ4_DEPRECATED("use LZ4_saveDictHC() instead") LZ4LIB_API char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") LZ4LIB_API int LZ4_freeHC (void* LZ4HC_Data);
-LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
+LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API int LZ4_sizeofStreamStateHC(void);
-LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") LZ4LIB_API int LZ4_resetStreamStateHC(void* state, char* inputBuffer);
+LZ4_DEPRECATED("use LZ4_initStreamHC() instead") LZ4LIB_API int LZ4_resetStreamStateHC(void* state, char* inputBuffer);
+
+
+/* LZ4_resetStreamHC() is now replaced by LZ4_initStreamHC().
+ * The intention is to emphasize the difference with LZ4_resetStreamHC_fast(),
+ * which is now the recommended function to start a new stream of blocks,
+ * but cannot be used to initialize a memory segment containing arbitrary garbage data.
+ *
+ * It is recommended to switch to LZ4_initStreamHC().
+ * LZ4_resetStreamHC() will generate deprecation warnings in a future version.
+ */
+LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionLevel);
#if defined (__cplusplus)
@@ -250,44 +340,22 @@ LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") LZ4LIB_API int LZ4_resetStr
extern "C" {
#endif
-/*! LZ4_compress_HC_destSize() : v1.8.0 (experimental)
- * Will try to compress as much data from `src` as possible
- * that can fit into `targetDstSize` budget.
- * Result is provided in 2 parts :
- * @return : the number of bytes written into 'dst'
- * or 0 if compression fails.
- * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src`
- */
-int LZ4_compress_HC_destSize(void* LZ4HC_Data,
- const char* src, char* dst,
- int* srcSizePtr, int targetDstSize,
- int compressionLevel);
-
-/*! LZ4_compress_HC_continue_destSize() : v1.8.0 (experimental)
- * Similar as LZ4_compress_HC_continue(),
- * but will read a variable nb of bytes from `src`
- * to fit into `targetDstSize` budget.
- * Result is provided in 2 parts :
- * @return : the number of bytes written into 'dst'
- * or 0 if compression fails.
- * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src`.
- */
-int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr,
- const char* src, char* dst,
- int* srcSizePtr, int targetDstSize);
-
-/*! LZ4_setCompressionLevel() : v1.8.0 (experimental)
- * It's possible to change compression level between 2 invocations of LZ4_compress_HC_continue*()
+/*! LZ4_setCompressionLevel() : v1.8.0+ (experimental)
+ * It's possible to change compression level
+ * between successive invocations of LZ4_compress_HC_continue*()
+ * for dynamic adaptation.
*/
-void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
+LZ4LIB_STATIC_API void LZ4_setCompressionLevel(
+ LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
-/*! LZ4_favorDecompressionSpeed() : v1.8.2 (experimental)
- * Parser will select decisions favoring decompression over compression ratio.
- * Only work at highest compression settings (level >= LZ4HC_CLEVEL_OPT_MIN)
+/*! LZ4_favorDecompressionSpeed() : v1.8.2+ (experimental)
+ * Opt. Parser will favor decompression speed over compression ratio.
+ * Only applicable to levels >= LZ4HC_CLEVEL_OPT_MIN.
*/
-void LZ4_favorDecompressionSpeed(LZ4_streamHC_t* LZ4_streamHCPtr, int favor);
+LZ4LIB_STATIC_API void LZ4_favorDecompressionSpeed(
+ LZ4_streamHC_t* LZ4_streamHCPtr, int favor);
-/*! LZ4_resetStreamHC_fast() :
+/*! LZ4_resetStreamHC_fast() : v1.9.0+
* When an LZ4_streamHC_t is known to be in a internally coherent state,
* it can often be prepared for a new compression with almost no work, only
* sometimes falling back to the full, expensive reset that is always required
@@ -304,8 +372,14 @@ void LZ4_favorDecompressionSpeed(LZ4_streamHC_t* LZ4_streamHCPtr, int favor);
* - the stream was in an indeterminate state and was used in a compression
* call that fully reset the state (LZ4_compress_HC_extStateHC()) and that
* returned success
+ *
+ * Note:
+ * A stream that was last used in a compression call that returned an error
+ * may be passed to this function. However, it will be fully reset, which will
+ * clear any existing history and settings from the context.
*/
-void LZ4_resetStreamHC_fast(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
+LZ4LIB_STATIC_API void LZ4_resetStreamHC_fast(
+ LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
/*! LZ4_compress_HC_extStateHC_fastReset() :
* A variant of LZ4_compress_HC_extStateHC().
@@ -318,7 +392,11 @@ void LZ4_resetStreamHC_fast(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLeve
* LZ4_resetStreamHC_fast() while LZ4_compress_HC_extStateHC() starts with a
* call to LZ4_resetStreamHC().
*/
-int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel);
+LZ4LIB_STATIC_API int LZ4_compress_HC_extStateHC_fastReset (
+ void* state,
+ const char* src, char* dst,
+ int srcSize, int dstCapacity,
+ int compressionLevel);
/*! LZ4_attach_HC_dictionary() :
* This is an experimental API that allows for the efficient use of a
@@ -345,7 +423,9 @@ int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* ds
* stream (and source buffer) must remain in-place / accessible / unchanged
* through the lifetime of the stream session.
*/
-LZ4LIB_API void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC_t *dictionary_stream);
+LZ4LIB_STATIC_API void LZ4_attach_HC_dictionary(
+ LZ4_streamHC_t *working_stream,
+ const LZ4_streamHC_t *dictionary_stream);
#if defined (__cplusplus)
}
diff --git a/lib/xxhash.c b/lib/xxhash.c
index 3fc97fd..ff28749 100644
--- a/lib/xxhash.c
+++ b/lib/xxhash.c
@@ -50,20 +50,26 @@
* Prefer these methods in priority order (0 > 1 > 2)
*/
#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
-# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
+# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) \
+ || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) \
+ || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
# define XXH_FORCE_MEMORY_ACCESS 2
# elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || \
- (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
+ (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) \
+ || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) \
+ || defined(__ARM_ARCH_7S__) ))
# define XXH_FORCE_MEMORY_ACCESS 1
# endif
#endif
/*!XXH_ACCEPT_NULL_INPUT_POINTER :
- * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer.
- * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input.
- * By default, this option is disabled. To enable it, uncomment below define :
+ * If input pointer is NULL, xxHash default behavior is to dereference it, triggering a segfault.
+ * When this macro is enabled, xxHash actively checks input for null pointer.
+ * It it is, result for null input pointers is the same as a null-length input.
*/
-/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */
+#ifndef XXH_ACCEPT_NULL_INPUT_POINTER /* can be defined externally */
+# define XXH_ACCEPT_NULL_INPUT_POINTER 0
+#endif
/*!XXH_FORCE_NATIVE_FORMAT :
* By default, xxHash library provides endian-independent Hash values, based on little-endian convention.
@@ -80,8 +86,9 @@
/*!XXH_FORCE_ALIGN_CHECK :
* This is a minor performance trick, only useful with lots of very small keys.
* It means : check for aligned/unaligned input.
- * The check costs one initial branch per hash; set to 0 when the input data
- * is guaranteed to be aligned.
+ * The check costs one initial branch per hash;
+ * set it to 0 when the input is guaranteed to be aligned,
+ * or when alignment doesn't matter for performance.
*/
#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */
# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
@@ -104,6 +111,8 @@ static void XXH_free (void* p) { free(p); }
#include <string.h>
static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
+#include <assert.h> /* assert */
+
#define XXH_STATIC_LINKING_ONLY
#include "xxhash.h"
@@ -113,40 +122,35 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp
***************************************/
#ifdef _MSC_VER /* Visual Studio */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
-#endif
-
-#ifndef XXH_FORCE_INLINE
-# ifdef _MSC_VER /* Visual Studio */
-# define XXH_FORCE_INLINE static __forceinline
-# else
-# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
-# ifdef __GNUC__
-# define XXH_FORCE_INLINE static inline __attribute__((always_inline))
-# else
-# define XXH_FORCE_INLINE static inline
-# endif
+# define FORCE_INLINE static __forceinline
+#else
+# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
+# ifdef __GNUC__
+# define FORCE_INLINE static inline __attribute__((always_inline))
# else
-# define XXH_FORCE_INLINE static
-# endif /* __STDC_VERSION__ */
-# endif /* _MSC_VER */
-#endif /* XXH_FORCE_INLINE */
+# define FORCE_INLINE static inline
+# endif
+# else
+# define FORCE_INLINE static
+# endif /* __STDC_VERSION__ */
+#endif
/* *************************************
* Basic Types
***************************************/
#ifndef MEM_MODULE
-# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+# if !defined (__VMS) \
+ && (defined (__cplusplus) \
+ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
- typedef int32_t S32;
# else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
- typedef signed int S32;
# endif
#endif
@@ -213,8 +217,12 @@ typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */
#ifndef XXH_CPU_LITTLE_ENDIAN
- static const int g_one = 1;
-# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one))
+static int XXH_isLittleEndian(void)
+{
+ const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
+ return one.c[0];
+}
+# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian()
#endif
@@ -223,7 +231,7 @@ typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
*****************************/
typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
-XXH_FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
+FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
{
if (align==XXH_unaligned)
return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
@@ -231,7 +239,7 @@ XXH_FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, X
return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr);
}
-XXH_FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
+FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
{
return XXH_readLE32_align(ptr, endian, XXH_unaligned);
}
@@ -245,12 +253,12 @@ static U32 XXH_readBE32(const void* ptr)
/* *************************************
* Macros
***************************************/
-#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
+#define XXH_STATIC_ASSERT(c) { enum { XXH_sa = 1/(int)(!!(c)) }; } /* use after variable declarations */
XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
/* *******************************************************************
-* 32-bits hash functions
+* 32-bit hash functions
*********************************************************************/
static const U32 PRIME32_1 = 2654435761U;
static const U32 PRIME32_2 = 2246822519U;
@@ -266,14 +274,89 @@ static U32 XXH32_round(U32 seed, U32 input)
return seed;
}
-XXH_FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align)
+/* mix all bits */
+static U32 XXH32_avalanche(U32 h32)
+{
+ h32 ^= h32 >> 15;
+ h32 *= PRIME32_2;
+ h32 ^= h32 >> 13;
+ h32 *= PRIME32_3;
+ h32 ^= h32 >> 16;
+ return(h32);
+}
+
+#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align)
+
+static U32
+XXH32_finalize(U32 h32, const void* ptr, size_t len,
+ XXH_endianess endian, XXH_alignment align)
+
+{
+ const BYTE* p = (const BYTE*)ptr;
+
+#define PROCESS1 \
+ h32 += (*p++) * PRIME32_5; \
+ h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;
+
+#define PROCESS4 \
+ h32 += XXH_get32bits(p) * PRIME32_3; \
+ p+=4; \
+ h32 = XXH_rotl32(h32, 17) * PRIME32_4 ;
+
+ switch(len&15) /* or switch(bEnd - p) */
+ {
+ case 12: PROCESS4;
+ /* fallthrough */
+ case 8: PROCESS4;
+ /* fallthrough */
+ case 4: PROCESS4;
+ return XXH32_avalanche(h32);
+
+ case 13: PROCESS4;
+ /* fallthrough */
+ case 9: PROCESS4;
+ /* fallthrough */
+ case 5: PROCESS4;
+ PROCESS1;
+ return XXH32_avalanche(h32);
+
+ case 14: PROCESS4;
+ /* fallthrough */
+ case 10: PROCESS4;
+ /* fallthrough */
+ case 6: PROCESS4;
+ PROCESS1;
+ PROCESS1;
+ return XXH32_avalanche(h32);
+
+ case 15: PROCESS4;
+ /* fallthrough */
+ case 11: PROCESS4;
+ /* fallthrough */
+ case 7: PROCESS4;
+ /* fallthrough */
+ case 3: PROCESS1;
+ /* fallthrough */
+ case 2: PROCESS1;
+ /* fallthrough */
+ case 1: PROCESS1;
+ /* fallthrough */
+ case 0: return XXH32_avalanche(h32);
+ }
+ assert(0);
+ return h32; /* reaching this point is deemed impossible */
+}
+
+
+FORCE_INLINE U32
+XXH32_endian_align(const void* input, size_t len, U32 seed,
+ XXH_endianess endian, XXH_alignment align)
{
const BYTE* p = (const BYTE*)input;
const BYTE* bEnd = p + len;
U32 h32;
-#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align)
-#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)
if (p==NULL) {
len=0;
bEnd=p=(const BYTE*)(size_t)16;
@@ -281,7 +364,7 @@ XXH_FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed,
#endif
if (len>=16) {
- const BYTE* const limit = bEnd - 16;
+ const BYTE* const limit = bEnd - 15;
U32 v1 = seed + PRIME32_1 + PRIME32_2;
U32 v2 = seed + PRIME32_2;
U32 v3 = seed + 0;
@@ -292,34 +375,17 @@ XXH_FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed,
v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4;
v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4;
v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4;
- } while (p<=limit);
+ } while (p < limit);
- h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
+ h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7)
+ + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
} else {
h32 = seed + PRIME32_5;
}
- h32 += (U32) len;
-
- while (p+4<=bEnd) {
- h32 += XXH_get32bits(p) * PRIME32_3;
- h32 = XXH_rotl32(h32, 17) * PRIME32_4 ;
- p+=4;
- }
-
- while (p<bEnd) {
- h32 += (*p) * PRIME32_5;
- h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;
- p++;
- }
-
- h32 ^= h32 >> 15;
- h32 *= PRIME32_2;
- h32 ^= h32 >> 13;
- h32 *= PRIME32_3;
- h32 ^= h32 >> 16;
+ h32 += (U32)len;
- return h32;
+ return XXH32_finalize(h32, p, len&15, endian, align);
}
@@ -371,74 +437,81 @@ XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t
XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed)
{
XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
- memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */
+ memset(&state, 0, sizeof(state));
state.v1 = seed + PRIME32_1 + PRIME32_2;
state.v2 = seed + PRIME32_2;
state.v3 = seed + 0;
state.v4 = seed - PRIME32_1;
- memcpy(statePtr, &state, sizeof(state));
+ /* do not write into reserved, planned to be removed in a future version */
+ memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved));
return XXH_OK;
}
-XXH_FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian)
+FORCE_INLINE XXH_errorcode
+XXH32_update_endian(XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian)
{
- const BYTE* p = (const BYTE*)input;
- const BYTE* const bEnd = p + len;
-
-#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
- if (input==NULL) return XXH_ERROR;
+ if (input==NULL)
+#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)
+ return XXH_OK;
+#else
+ return XXH_ERROR;
#endif
- state->total_len_32 += (unsigned)len;
- state->large_len |= (len>=16) | (state->total_len_32>=16);
+ { const BYTE* p = (const BYTE*)input;
+ const BYTE* const bEnd = p + len;
- if (state->memsize + len < 16) { /* fill in tmp buffer */
- XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len);
- state->memsize += (unsigned)len;
- return XXH_OK;
- }
+ state->total_len_32 += (unsigned)len;
+ state->large_len |= (len>=16) | (state->total_len_32>=16);
- if (state->memsize) { /* some data left from previous update */
- XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize);
- { const U32* p32 = state->mem32;
- state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++;
- state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++;
- state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++;
- state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++;
+ if (state->memsize + len < 16) { /* fill in tmp buffer */
+ XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len);
+ state->memsize += (unsigned)len;
+ return XXH_OK;
}
- p += 16-state->memsize;
- state->memsize = 0;
- }
-
- if (p <= bEnd-16) {
- const BYTE* const limit = bEnd - 16;
- U32 v1 = state->v1;
- U32 v2 = state->v2;
- U32 v3 = state->v3;
- U32 v4 = state->v4;
- do {
- v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4;
- v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4;
- v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4;
- v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4;
- } while (p<=limit);
+ if (state->memsize) { /* some data left from previous update */
+ XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize);
+ { const U32* p32 = state->mem32;
+ state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++;
+ state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++;
+ state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++;
+ state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian));
+ }
+ p += 16-state->memsize;
+ state->memsize = 0;
+ }
- state->v1 = v1;
- state->v2 = v2;
- state->v3 = v3;
- state->v4 = v4;
- }
+ if (p <= bEnd-16) {
+ const BYTE* const limit = bEnd - 16;
+ U32 v1 = state->v1;
+ U32 v2 = state->v2;
+ U32 v3 = state->v3;
+ U32 v4 = state->v4;
+
+ do {
+ v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4;
+ v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4;
+ v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4;
+ v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4;
+ } while (p<=limit);
+
+ state->v1 = v1;
+ state->v2 = v2;
+ state->v3 = v3;
+ state->v4 = v4;
+ }
- if (p < bEnd) {
- XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
- state->memsize = (unsigned)(bEnd-p);
+ if (p < bEnd) {
+ XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
+ state->memsize = (unsigned)(bEnd-p);
+ }
}
return XXH_OK;
}
+
XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len)
{
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
@@ -450,40 +523,23 @@ XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void*
}
-
-XXH_FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian)
+FORCE_INLINE U32
+XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian)
{
- const BYTE * p = (const BYTE*)state->mem32;
- const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize;
U32 h32;
if (state->large_len) {
- h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18);
+ h32 = XXH_rotl32(state->v1, 1)
+ + XXH_rotl32(state->v2, 7)
+ + XXH_rotl32(state->v3, 12)
+ + XXH_rotl32(state->v4, 18);
} else {
h32 = state->v3 /* == seed */ + PRIME32_5;
}
h32 += state->total_len_32;
- while (p+4<=bEnd) {
- h32 += XXH_readLE32(p, endian) * PRIME32_3;
- h32 = XXH_rotl32(h32, 17) * PRIME32_4;
- p+=4;
- }
-
- while (p<bEnd) {
- h32 += (*p) * PRIME32_5;
- h32 = XXH_rotl32(h32, 11) * PRIME32_1;
- p++;
- }
-
- h32 ^= h32 >> 15;
- h32 *= PRIME32_2;
- h32 ^= h32 >> 13;
- h32 *= PRIME32_3;
- h32 ^= h32 >> 16;
-
- return h32;
+ return XXH32_finalize(h32, state->mem32, state->memsize, endian, XXH_aligned);
}
@@ -503,7 +559,7 @@ XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in)
/*! Default XXH result types are basic unsigned 32 and 64 bits.
* The canonical representation follows human-readable write convention, aka big-endian (large digits first).
* These functions allow transformation of hash result into and from its canonical format.
-* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs.
+* This way, hash values can be written into a file or buffer, remaining comparable across different systems.
*/
XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
@@ -522,18 +578,21 @@ XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src
#ifndef XXH_NO_LONG_LONG
/* *******************************************************************
-* 64-bits hash functions
+* 64-bit hash functions
*********************************************************************/
/*====== Memory access ======*/
#ifndef MEM_MODULE
# define MEM_MODULE
-# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+# if !defined (__VMS) \
+ && (defined (__cplusplus) \
+ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
# include <stdint.h>
typedef uint64_t U64;
# else
- typedef unsigned long long U64; /* if your compiler doesn't support unsigned long long, replace by another 64-bit type here. Note that xxhash.h will also need to be updated. */
+ /* if compiler doesn't support unsigned long long, replace by another 64-bit type */
+ typedef unsigned long long U64;
# endif
#endif
@@ -583,7 +642,7 @@ static U64 XXH_swap64 (U64 x)
}
#endif
-XXH_FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
+FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
{
if (align==XXH_unaligned)
return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
@@ -591,7 +650,7 @@ XXH_FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, X
return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr);
}
-XXH_FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
+FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
{
return XXH_readLE64_align(ptr, endian, XXH_unaligned);
}
@@ -626,14 +685,137 @@ static U64 XXH64_mergeRound(U64 acc, U64 val)
return acc;
}
-XXH_FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align)
+static U64 XXH64_avalanche(U64 h64)
+{
+ h64 ^= h64 >> 33;
+ h64 *= PRIME64_2;
+ h64 ^= h64 >> 29;
+ h64 *= PRIME64_3;
+ h64 ^= h64 >> 32;
+ return h64;
+}
+
+
+#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align)
+
+static U64
+XXH64_finalize(U64 h64, const void* ptr, size_t len,
+ XXH_endianess endian, XXH_alignment align)
+{
+ const BYTE* p = (const BYTE*)ptr;
+
+#define PROCESS1_64 \
+ h64 ^= (*p++) * PRIME64_5; \
+ h64 = XXH_rotl64(h64, 11) * PRIME64_1;
+
+#define PROCESS4_64 \
+ h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; \
+ p+=4; \
+ h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
+
+#define PROCESS8_64 { \
+ U64 const k1 = XXH64_round(0, XXH_get64bits(p)); \
+ p+=8; \
+ h64 ^= k1; \
+ h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; \
+}
+
+ switch(len&31) {
+ case 24: PROCESS8_64;
+ /* fallthrough */
+ case 16: PROCESS8_64;
+ /* fallthrough */
+ case 8: PROCESS8_64;
+ return XXH64_avalanche(h64);
+
+ case 28: PROCESS8_64;
+ /* fallthrough */
+ case 20: PROCESS8_64;
+ /* fallthrough */
+ case 12: PROCESS8_64;
+ /* fallthrough */
+ case 4: PROCESS4_64;
+ return XXH64_avalanche(h64);
+
+ case 25: PROCESS8_64;
+ /* fallthrough */
+ case 17: PROCESS8_64;
+ /* fallthrough */
+ case 9: PROCESS8_64;
+ PROCESS1_64;
+ return XXH64_avalanche(h64);
+
+ case 29: PROCESS8_64;
+ /* fallthrough */
+ case 21: PROCESS8_64;
+ /* fallthrough */
+ case 13: PROCESS8_64;
+ /* fallthrough */
+ case 5: PROCESS4_64;
+ PROCESS1_64;
+ return XXH64_avalanche(h64);
+
+ case 26: PROCESS8_64;
+ /* fallthrough */
+ case 18: PROCESS8_64;
+ /* fallthrough */
+ case 10: PROCESS8_64;
+ PROCESS1_64;
+ PROCESS1_64;
+ return XXH64_avalanche(h64);
+
+ case 30: PROCESS8_64;
+ /* fallthrough */
+ case 22: PROCESS8_64;
+ /* fallthrough */
+ case 14: PROCESS8_64;
+ /* fallthrough */
+ case 6: PROCESS4_64;
+ PROCESS1_64;
+ PROCESS1_64;
+ return XXH64_avalanche(h64);
+
+ case 27: PROCESS8_64;
+ /* fallthrough */
+ case 19: PROCESS8_64;
+ /* fallthrough */
+ case 11: PROCESS8_64;
+ PROCESS1_64;
+ PROCESS1_64;
+ PROCESS1_64;
+ return XXH64_avalanche(h64);
+
+ case 31: PROCESS8_64;
+ /* fallthrough */
+ case 23: PROCESS8_64;
+ /* fallthrough */
+ case 15: PROCESS8_64;
+ /* fallthrough */
+ case 7: PROCESS4_64;
+ /* fallthrough */
+ case 3: PROCESS1_64;
+ /* fallthrough */
+ case 2: PROCESS1_64;
+ /* fallthrough */
+ case 1: PROCESS1_64;
+ /* fallthrough */
+ case 0: return XXH64_avalanche(h64);
+ }
+
+ /* impossible to reach */
+ assert(0);
+ return 0; /* unreachable, but some compilers complain without it */
+}
+
+FORCE_INLINE U64
+XXH64_endian_align(const void* input, size_t len, U64 seed,
+ XXH_endianess endian, XXH_alignment align)
{
const BYTE* p = (const BYTE*)input;
- const BYTE* const bEnd = p + len;
+ const BYTE* bEnd = p + len;
U64 h64;
-#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align)
-#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)
if (p==NULL) {
len=0;
bEnd=p=(const BYTE*)(size_t)32;
@@ -666,32 +848,7 @@ XXH_FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed,
h64 += (U64) len;
- while (p+8<=bEnd) {
- U64 const k1 = XXH64_round(0, XXH_get64bits(p));
- h64 ^= k1;
- h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
- p+=8;
- }
-
- if (p+4<=bEnd) {
- h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1;
- h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
- p+=4;
- }
-
- while (p<bEnd) {
- h64 ^= (*p) * PRIME64_5;
- h64 = XXH_rotl64(h64, 11) * PRIME64_1;
- p++;
- }
-
- h64 ^= h64 >> 33;
- h64 *= PRIME64_2;
- h64 ^= h64 >> 29;
- h64 *= PRIME64_3;
- h64 ^= h64 >> 32;
-
- return h64;
+ return XXH64_finalize(h64, p, len, endian, align);
}
@@ -741,65 +898,71 @@ XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t
XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed)
{
XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
- memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */
+ memset(&state, 0, sizeof(state));
state.v1 = seed + PRIME64_1 + PRIME64_2;
state.v2 = seed + PRIME64_2;
state.v3 = seed + 0;
state.v4 = seed - PRIME64_1;
- memcpy(statePtr, &state, sizeof(state));
+ /* do not write into reserved, planned to be removed in a future version */
+ memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved));
return XXH_OK;
}
-XXH_FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian)
+FORCE_INLINE XXH_errorcode
+XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian)
{
- const BYTE* p = (const BYTE*)input;
- const BYTE* const bEnd = p + len;
-
-#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
- if (input==NULL) return XXH_ERROR;
+ if (input==NULL)
+#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)
+ return XXH_OK;
+#else
+ return XXH_ERROR;
#endif
- state->total_len += len;
-
- if (state->memsize + len < 32) { /* fill in tmp buffer */
- XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len);
- state->memsize += (U32)len;
- return XXH_OK;
- }
+ { const BYTE* p = (const BYTE*)input;
+ const BYTE* const bEnd = p + len;
- if (state->memsize) { /* tmp buffer is full */
- XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize);
- state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian));
- state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian));
- state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian));
- state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian));
- p += 32-state->memsize;
- state->memsize = 0;
- }
+ state->total_len += len;
- if (p+32 <= bEnd) {
- const BYTE* const limit = bEnd - 32;
- U64 v1 = state->v1;
- U64 v2 = state->v2;
- U64 v3 = state->v3;
- U64 v4 = state->v4;
+ if (state->memsize + len < 32) { /* fill in tmp buffer */
+ XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len);
+ state->memsize += (U32)len;
+ return XXH_OK;
+ }
- do {
- v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8;
- v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8;
- v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8;
- v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8;
- } while (p<=limit);
+ if (state->memsize) { /* tmp buffer is full */
+ XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize);
+ state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian));
+ state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian));
+ state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian));
+ state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian));
+ p += 32-state->memsize;
+ state->memsize = 0;
+ }
- state->v1 = v1;
- state->v2 = v2;
- state->v3 = v3;
- state->v4 = v4;
- }
+ if (p+32 <= bEnd) {
+ const BYTE* const limit = bEnd - 32;
+ U64 v1 = state->v1;
+ U64 v2 = state->v2;
+ U64 v3 = state->v3;
+ U64 v4 = state->v4;
+
+ do {
+ v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8;
+ v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8;
+ v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8;
+ v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8;
+ } while (p<=limit);
+
+ state->v1 = v1;
+ state->v2 = v2;
+ state->v3 = v3;
+ state->v4 = v4;
+ }
- if (p < bEnd) {
- XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
- state->memsize = (unsigned)(bEnd-p);
+ if (p < bEnd) {
+ XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
+ state->memsize = (unsigned)(bEnd-p);
+ }
}
return XXH_OK;
@@ -815,10 +978,8 @@ XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void*
return XXH64_update_endian(state_in, input, len, XXH_bigEndian);
}
-XXH_FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian)
+FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian)
{
- const BYTE * p = (const BYTE*)state->mem64;
- const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize;
U64 h64;
if (state->total_len >= 32) {
@@ -833,37 +994,12 @@ XXH_FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endian
h64 = XXH64_mergeRound(h64, v3);
h64 = XXH64_mergeRound(h64, v4);
} else {
- h64 = state->v3 + PRIME64_5;
+ h64 = state->v3 /*seed*/ + PRIME64_5;
}
h64 += (U64) state->total_len;
- while (p+8<=bEnd) {
- U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian));
- h64 ^= k1;
- h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
- p+=8;
- }
-
- if (p+4<=bEnd) {
- h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1;
- h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
- p+=4;
- }
-
- while (p<bEnd) {
- h64 ^= (*p) * PRIME64_5;
- h64 = XXH_rotl64(h64, 11) * PRIME64_1;
- p++;
- }
-
- h64 ^= h64 >> 33;
- h64 *= PRIME64_2;
- h64 ^= h64 >> 29;
- h64 *= PRIME64_3;
- h64 ^= h64 >> 32;
-
- return h64;
+ return XXH64_finalize(h64, state->mem64, (size_t)state->total_len, endian, XXH_aligned);
}
XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in)
diff --git a/lib/xxhash.h b/lib/xxhash.h
index 870a6d9..d6bad94 100644
--- a/lib/xxhash.h
+++ b/lib/xxhash.h
@@ -57,8 +57,8 @@ Q.Score is a measure of quality of the hash function.
It depends on successfully passing SMHasher test set.
10 is a perfect score.
-A 64-bits version, named XXH64, is available since r35.
-It offers much better speed, but for 64-bits applications only.
+A 64-bit version, named XXH64, is available since r35.
+It offers much better speed, but for 64-bit applications only.
Name Speed on 64 bits Speed on 32 bits
XXH64 13.8 GB/s 1.9 GB/s
XXH32 6.8 GB/s 6.0 GB/s
@@ -80,18 +80,19 @@ typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
/* ****************************
-* API modifier
-******************************/
-/** XXH_PRIVATE_API
-* This is useful to include xxhash functions in `static` mode
-* in order to inline them, and remove their symbol from the public list.
-* Methodology :
-* #define XXH_PRIVATE_API
-* #include "xxhash.h"
-* `xxhash.c` is automatically included.
-* It's not useful to compile and link it as a separate module.
-*/
-#ifdef XXH_PRIVATE_API
+ * API modifier
+ ******************************/
+/** XXH_INLINE_ALL (and XXH_PRIVATE_API)
+ * This is useful to include xxhash functions in `static` mode
+ * in order to inline them, and remove their symbol from the public list.
+ * Inlining can offer dramatic performance improvement on small keys.
+ * Methodology :
+ * #define XXH_INLINE_ALL
+ * #include "xxhash.h"
+ * `xxhash.c` is automatically included.
+ * It's not useful to compile and link it as a separate module.
+ */
+#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)
# ifndef XXH_STATIC_LINKING_ONLY
# define XXH_STATIC_LINKING_ONLY
# endif
@@ -102,23 +103,24 @@ typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
# elif defined(_MSC_VER)
# define XXH_PUBLIC_API static __inline
# else
-# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */
+ /* this version may generate warnings for unused static functions */
+# define XXH_PUBLIC_API static
# endif
#else
# define XXH_PUBLIC_API /* do nothing */
-#endif /* XXH_PRIVATE_API */
-
-/*!XXH_NAMESPACE, aka Namespace Emulation :
-
-If you want to include _and expose_ xxHash functions from within your own library,
-but also want to avoid symbol collisions with other libraries which may also include xxHash,
-
-you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library
-with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values).
-
-Note that no change is required within the calling program as long as it includes `xxhash.h` :
-regular symbol name will be automatically translated by this header.
-*/
+#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */
+
+/*! XXH_NAMESPACE, aka Namespace Emulation :
+ *
+ * If you want to include _and expose_ xxHash functions from within your own library,
+ * but also want to avoid symbol collisions with other libraries which may also include xxHash,
+ *
+ * you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library
+ * with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values).
+ *
+ * Note that no change is required within the calling program as long as it includes `xxhash.h` :
+ * regular symbol name will be automatically translated by this header.
+ */
#ifdef XXH_NAMESPACE
# define XXH_CAT(A,B) A##B
# define XXH_NAME2(A,B) XXH_CAT(A,B)
@@ -149,18 +151,18 @@ regular symbol name will be automatically translated by this header.
***************************************/
#define XXH_VERSION_MAJOR 0
#define XXH_VERSION_MINOR 6
-#define XXH_VERSION_RELEASE 2
+#define XXH_VERSION_RELEASE 5
#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
XXH_PUBLIC_API unsigned XXH_versionNumber (void);
/*-**********************************************************************
-* 32-bits hash
+* 32-bit hash
************************************************************************/
-typedef unsigned int XXH32_hash_t;
+typedef unsigned int XXH32_hash_t;
/*! XXH32() :
- Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input".
+ Calculate the 32-bit hash of sequence "length" bytes stored at memory address "input".
The memory between input & input+length must be valid (allocated and read-accessible).
"seed" can be used to alter the result predictably.
Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s */
@@ -177,26 +179,25 @@ XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void*
XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr);
/*
-These functions generate the xxHash of an input provided in multiple segments.
-Note that, for small input, they are slower than single-call functions, due to state management.
-For small input, prefer `XXH32()` and `XXH64()` .
-
-XXH state must first be allocated, using XXH*_createState() .
-
-Start a new hash by initializing state with a seed, using XXH*_reset().
-
-Then, feed the hash state by calling XXH*_update() as many times as necessary.
-Obviously, input must be allocated and read accessible.
-The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
-
-Finally, a hash value can be produced anytime, by using XXH*_digest().
-This function returns the nn-bits hash as an int or long long.
-
-It's still possible to continue inserting input into the hash state after a digest,
-and generate some new hashes later on, by calling again XXH*_digest().
-
-When done, free XXH state space if it was allocated dynamically.
-*/
+ * Streaming functions generate the xxHash of an input provided in multiple segments.
+ * Note that, for small input, they are slower than single-call functions, due to state management.
+ * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized.
+ *
+ * XXH state must first be allocated, using XXH*_createState() .
+ *
+ * Start a new hash by initializing state with a seed, using XXH*_reset().
+ *
+ * Then, feed the hash state by calling XXH*_update() as many times as necessary.
+ * The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
+ *
+ * Finally, a hash value can be produced anytime, by using XXH*_digest().
+ * This function returns the nn-bits hash as an int or long long.
+ *
+ * It's still possible to continue inserting input into the hash state after a digest,
+ * and generate some new hashes later on, by calling again XXH*_digest().
+ *
+ * When done, free XXH state space if it was allocated dynamically.
+ */
/*====== Canonical representation ======*/
@@ -205,22 +206,22 @@ XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t
XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.
-* The canonical representation uses human-readable write convention, aka big-endian (large digits first).
-* These functions allow transformation of hash result into and from its canonical format.
-* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
-*/
+ * The canonical representation uses human-readable write convention, aka big-endian (large digits first).
+ * These functions allow transformation of hash result into and from its canonical format.
+ * This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
+ */
#ifndef XXH_NO_LONG_LONG
/*-**********************************************************************
-* 64-bits hash
+* 64-bit hash
************************************************************************/
typedef unsigned long long XXH64_hash_t;
/*! XXH64() :
- Calculate the 64-bits hash of sequence of length "len" stored at memory address "input".
+ Calculate the 64-bit hash of sequence of length "len" stored at memory address "input".
"seed" can be used to alter the result predictably.
- This function runs faster on 64-bits systems, but slower on 32-bits systems (see benchmark).
+ This function runs faster on 64-bit systems, but slower on 32-bit systems (see benchmark).
*/
XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed);
@@ -241,48 +242,82 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src
#endif /* XXH_NO_LONG_LONG */
+
#ifdef XXH_STATIC_LINKING_ONLY
/* ================================================================================================
- This section contains definitions which are not guaranteed to remain stable.
+ This section contains declarations which are not guaranteed to remain stable.
They may change in future versions, becoming incompatible with a different version of the library.
- They shall only be used with static linking.
- Never use these definitions in association with dynamic linking !
+ These declarations should only be used with static linking.
+ Never use them in association with dynamic linking !
=================================================================================================== */
-/* These definitions are only meant to allow allocation of XXH state
- statically, on stack, or in a struct for example.
- Do not use members directly. */
-
- struct XXH32_state_s {
- unsigned total_len_32;
- unsigned large_len;
- unsigned v1;
- unsigned v2;
- unsigned v3;
- unsigned v4;
- unsigned mem32[4]; /* buffer defined as U32 for alignment */
- unsigned memsize;
- unsigned reserved; /* never read nor write, will be removed in a future version */
- }; /* typedef'd to XXH32_state_t */
-
-#ifndef XXH_NO_LONG_LONG
- struct XXH64_state_s {
- unsigned long long total_len;
- unsigned long long v1;
- unsigned long long v2;
- unsigned long long v3;
- unsigned long long v4;
- unsigned long long mem64[4]; /* buffer defined as U64 for alignment */
- unsigned memsize;
- unsigned reserved[2]; /* never read nor write, will be removed in a future version */
- }; /* typedef'd to XXH64_state_t */
+/* These definitions are only present to allow
+ * static allocation of XXH state, on stack or in a struct for example.
+ * Never **ever** use members directly. */
+
+#if !defined (__VMS) \
+ && (defined (__cplusplus) \
+ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+# include <stdint.h>
+
+struct XXH32_state_s {
+ uint32_t total_len_32;
+ uint32_t large_len;
+ uint32_t v1;
+ uint32_t v2;
+ uint32_t v3;
+ uint32_t v4;
+ uint32_t mem32[4];
+ uint32_t memsize;
+ uint32_t reserved; /* never read nor write, might be removed in a future version */
+}; /* typedef'd to XXH32_state_t */
+
+struct XXH64_state_s {
+ uint64_t total_len;
+ uint64_t v1;
+ uint64_t v2;
+ uint64_t v3;
+ uint64_t v4;
+ uint64_t mem64[4];
+ uint32_t memsize;
+ uint32_t reserved[2]; /* never read nor write, might be removed in a future version */
+}; /* typedef'd to XXH64_state_t */
+
+# else
+
+struct XXH32_state_s {
+ unsigned total_len_32;
+ unsigned large_len;
+ unsigned v1;
+ unsigned v2;
+ unsigned v3;
+ unsigned v4;
+ unsigned mem32[4];
+ unsigned memsize;
+ unsigned reserved; /* never read nor write, might be removed in a future version */
+}; /* typedef'd to XXH32_state_t */
+
+# ifndef XXH_NO_LONG_LONG /* remove 64-bit support */
+struct XXH64_state_s {
+ unsigned long long total_len;
+ unsigned long long v1;
+ unsigned long long v2;
+ unsigned long long v3;
+ unsigned long long v4;
+ unsigned long long mem64[4];
+ unsigned memsize;
+ unsigned reserved[2]; /* never read nor write, might be removed in a future version */
+}; /* typedef'd to XXH64_state_t */
+# endif
+
+# endif
+
+
+#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)
+# include "xxhash.c" /* include xxhash function bodies as `static`, for inlining */
#endif
-# ifdef XXH_PRIVATE_API
-# include "xxhash.c" /* include xxhash function bodies as `static`, for inlining */
-# endif
-
#endif /* XXH_STATIC_LINKING_ONLY */
generated by cgit v0.12 at 2024-11-16 21:05:36 (GMT)