summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--lib/Makefile6
-rw-r--r--lib/lz4.c30
-rw-r--r--lib/lz4.h12
-rw-r--r--lib/lz4frame.c15
-rw-r--r--lib/lz4frame.h120
-rw-r--r--lib/lz4frame_static.h117
-rw-r--r--lib/lz4hc.c223
-rw-r--r--lib/lz4hc.h83
-rw-r--r--programs/lz4io.c2
-rw-r--r--programs/util.h3
-rw-r--r--tests/Makefile3
-rw-r--r--tests/frametest.c6
-rw-r--r--tests/fuzzer.c75
13 files changed, 510 insertions, 185 deletions
diff --git a/lib/Makefile b/lib/Makefile
index 7b31239..bb45582 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -42,6 +42,7 @@ LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
LIBVER := $(shell echo $(LIBVER_SCRIPT))
+BUILD_SHARED:=yes
BUILD_STATIC:=yes
CPPFLAGS+= -DXXH_NAMESPACE=LZ4_
@@ -92,6 +93,7 @@ ifeq ($(BUILD_STATIC),yes) # can be disabled on command line
endif
$(LIBLZ4): $(SRCFILES)
+ifeq ($(BUILD_SHARED),yes) # can be disabled on command line
@echo compiling dynamic library $(LIBVER)
ifneq (,$(filter Windows%,$(OS)))
@$(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll\$@.dll
@@ -102,6 +104,7 @@ else
@ln -sf $@ liblz4.$(SHARED_EXT_MAJOR)
@ln -sf $@ liblz4.$(SHARED_EXT)
endif
+endif
liblz4: $(LIBLZ4)
@@ -163,9 +166,11 @@ ifeq ($(BUILD_STATIC),yes)
@$(INSTALL_DATA) liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
@$(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(INCLUDEDIR)/lz4frame_static.h
endif
+ifeq ($(BUILD_SHARED),yes)
@$(INSTALL_PROGRAM) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)
@ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
@ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
+endif
@echo Installing headers in $(INCLUDEDIR)
@$(INSTALL_DATA) lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h
@$(INSTALL_DATA) lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
@@ -181,6 +186,7 @@ uninstall:
@$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4.h
@$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
@$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4frame.h
+ @$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4frame_static.h
@echo lz4 libraries successfully uninstalled
endif
diff --git a/lib/lz4.c b/lib/lz4.c
index bb6b619..eaabe5b 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -776,9 +776,10 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
forwardH = LZ4_hashPosition(forwardIp, tableType);
LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
- if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue; /* match outside of valid area */
- if ((tableType != byU16) && (current - matchIndex > MAX_DISTANCE)) continue; /* too far - note: works even if matchIndex overflows */
- if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE); /* too_far presumed impossible with byU16 */
+ if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue; /* match outside of valid area */
+ assert(matchIndex < current);
+ if ((tableType != byU16) && (matchIndex+MAX_DISTANCE < current)) continue; /* too far */
+ if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE); /* too_far presumed impossible with byU16 */
if (LZ4_read32(match) == LZ4_read32(ip)) {
if (maybe_extMem) offset = current - matchIndex;
@@ -918,8 +919,9 @@ _next_match:
match = base + matchIndex;
}
LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
+ assert(matchIndex < current);
if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1)
- && ((tableType==byU16) ? 1 : (current - matchIndex <= MAX_DISTANCE))
+ && ((tableType==byU16) ? 1 : (matchIndex+MAX_DISTANCE >= current))
&& (LZ4_read32(match) == LZ4_read32(ip)) ) {
token=op++;
*token=0;
@@ -1304,7 +1306,11 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
DEBUGLOG(4, "LZ4_loadDict (%i bytes from %p into %p)", dictSize, dictionary, LZ4_dict);
- LZ4_prepareTable(dict, 0, tableType);
+ /* It's necessary to reset the context,
+ * and not just continue it with prepareTable()
+ * to avoid any risk of generating overflowing matchIndex
+ * when compressing using this dictionary */
+ LZ4_resetStream(LZ4_dict);
/* We always increment the offset by 64 KB, since, if the dict is longer,
* we truncate it to the last 64k, and if it's shorter, we still want to
@@ -1520,6 +1526,7 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => just decode everything */
if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */
if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1);
+ if ((endOnInput) && unlikely(srcSize==0)) return -1;
/* Main Loop : decode sequences */
while (1) {
@@ -1529,13 +1536,17 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
unsigned const token = *ip++;
+ assert(!endOnInput || ip <= iend); /* ip < iend before the increment */
/* shortcut for common case :
* in most circumstances, we expect to decode small matches (<= 18 bytes) separated by few literals (<= 14 bytes).
* this shortcut was tested on x86 and x64, where it improves decoding speed.
- * it has not yet been benchmarked on ARM, Power, mips, etc. */
- if (((ip + 14 /*maxLL*/ + 2 /*offset*/ <= iend)
- & (op + 14 /*maxLL*/ + 18 /*maxML*/ <= oend))
- & ((token < (15<<ML_BITS)) & ((token & ML_MASK) != 15)) ) {
+ * it has not yet been benchmarked on ARM, Power, mips, etc.
+ * NOTE: The loop begins with a read, so we must have one byte left at the end. */
+ if (endOnInput
+ && ((ip + 14 /*maxLL*/ + 2 /*offset*/ < iend)
+ & (op + 14 /*maxLL*/ + 18 /*maxML*/ <= oend)
+ & (token < (15<<ML_BITS))
+ & ((token & ML_MASK) != 15) ) ) {
size_t const ll = token >> ML_BITS;
size_t const off = LZ4_readLE16(ip+ll);
const BYTE* const matchPtr = op + ll - off; /* pointer underflow risk ? */
@@ -1553,6 +1564,7 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
/* decode literal length */
if ((length=(token>>ML_BITS)) == RUN_MASK) {
unsigned s;
+ if (unlikely(endOnInput ? ip >= iend-RUN_MASK : 0)) goto _output_error; /* overflow detection */
do {
s = *ip++;
length += s;
diff --git a/lib/lz4.h b/lib/lz4.h
index 0dfa19e..db6353c 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -206,15 +206,17 @@ LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePt
/*!
LZ4_decompress_fast() : **unsafe!**
This function is a bit faster than LZ4_decompress_safe(),
-but doesn't provide any security guarantee.
+but it may misbehave on malformed input because it doesn't perform full validation of compressed data.
originalSize : is the uncompressed size to regenerate
Destination buffer must be already allocated, and its size must be >= 'originalSize' bytes.
return : number of bytes read from source buffer (== compressed size).
If the source stream is detected malformed, the function stops decoding and return a negative result.
- note : This function respects memory boundaries for *properly formed* compressed data.
- However, it does not provide any protection against malicious input.
- It also doesn't know 'src' size, and implies it's >= compressed size.
- Use this function in trusted environment **only**.
+ note : This function is only usable if the originalSize of uncompressed data is known in advance.
+ The caller should also check that all the compressed input has been consumed properly,
+ i.e. that the return value matches the size of the buffer with compressed input.
+ The function never writes past the output buffer. However, since it doesn't know its 'src' size,
+ it may read past the intended input. Also, because match offsets are not validated during decoding,
+ reads from 'src' may underflow. Use this function in trusted environment **only**.
*/
LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 507e4fe..b616463 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -73,7 +73,8 @@ You can contact the author at :
/*-************************************
* Includes
**************************************/
-#include "lz4frame_static.h"
+#define LZ4F_STATIC_LINKING_ONLY
+#include "lz4frame.h"
#define LZ4_STATIC_LINKING_ONLY
#include "lz4.h"
#define LZ4_HC_STATIC_LINKING_ONLY
@@ -534,12 +535,8 @@ static void LZ4F_applyCDict(void* ctx,
LZ4_resetStream_fast((LZ4_stream_t *)ctx);
LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
} else {
- if (cdict) {
- memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx));
- LZ4_setCompressionLevel((LZ4_streamHC_t*)ctx, level);
- } else {
- LZ4_resetStreamHC((LZ4_streamHC_t*)(ctx), level);
- }
+ LZ4_resetStreamHC_fast((LZ4_streamHC_t*)ctx, level);
+ LZ4_attach_HC_dictionary((LZ4_streamHC_t *)ctx, cdict ? cdict->HCCtx : NULL);
}
}
@@ -725,11 +722,11 @@ static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, in
static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
{
+ LZ4F_applyCDict(ctx, cdict, level);
if (cdict) {
- LZ4F_applyCDict(ctx, cdict, level);
return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
}
- return LZ4_compress_HC_extStateHC(ctx, src, dst, srcSize, dstCapacity, level);
+ return LZ4_compress_HC_extStateHC_fastReset(ctx, src, dst, srcSize, dstCapacity, level);
}
static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index e80b1a1..bd715bd 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -409,3 +409,123 @@ LZ4FLIB_API void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx); /* always su
#endif
#endif /* LZ4F_H_09782039843 */
+
+#if defined(LZ4F_STATIC_LINKING_ONLY) && !defined(LZ4F_H_STATIC_09782039843)
+#define LZ4F_H_STATIC_09782039843
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/* These declarations are not stable and may change in the future. They are
+ * therefore only safe to depend on when the caller is statically linked
+ * against the library. To access their declarations, define
+ * LZ4F_STATIC_LINKING_ONLY.
+ *
+ * There is a further protection mechanism where these symbols aren't published
+ * into shared/dynamic libraries. You can override this behavior and force
+ * them to be published by defining LZ4F_PUBLISH_STATIC_FUNCTIONS. Use at
+ * your own risk.
+ */
+#ifdef LZ4F_PUBLISH_STATIC_FUNCTIONS
+#define LZ4FLIB_STATIC_API LZ4FLIB_API
+#else
+#define LZ4FLIB_STATIC_API
+#endif
+
+
+/* --- Error List --- */
+#define LZ4F_LIST_ERRORS(ITEM) \
+ ITEM(OK_NoError) \
+ ITEM(ERROR_GENERIC) \
+ ITEM(ERROR_maxBlockSize_invalid) \
+ ITEM(ERROR_blockMode_invalid) \
+ ITEM(ERROR_contentChecksumFlag_invalid) \
+ ITEM(ERROR_compressionLevel_invalid) \
+ ITEM(ERROR_headerVersion_wrong) \
+ ITEM(ERROR_blockChecksum_invalid) \
+ ITEM(ERROR_reservedFlag_set) \
+ ITEM(ERROR_allocation_failed) \
+ ITEM(ERROR_srcSize_tooLarge) \
+ ITEM(ERROR_dstMaxSize_tooSmall) \
+ ITEM(ERROR_frameHeader_incomplete) \
+ ITEM(ERROR_frameType_unknown) \
+ ITEM(ERROR_frameSize_wrong) \
+ ITEM(ERROR_srcPtr_wrong) \
+ ITEM(ERROR_decompressionFailed) \
+ ITEM(ERROR_headerChecksum_invalid) \
+ ITEM(ERROR_contentChecksum_invalid) \
+ ITEM(ERROR_frameDecoding_alreadyStarted) \
+ ITEM(ERROR_maxCode)
+
+#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM,
+
+/* enum list is exposed, to handle specific errors */
+typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes;
+
+LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult);
+
+
+
+/**********************************
+ * Bulk processing dictionary API
+ *********************************/
+typedef struct LZ4F_CDict_s LZ4F_CDict;
+
+/*! LZ4_createCDict() :
+ * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
+ * LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+ * LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
+ * `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict */
+LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize);
+LZ4FLIB_STATIC_API void LZ4F_freeCDict(LZ4F_CDict* CDict);
+
+
+/*! LZ4_compressFrame_usingCDict() :
+ * Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary.
+ * cctx must point to a context created by LZ4F_createCompressionContext().
+ * If cdict==NULL, compress without a dictionary.
+ * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
+ * If this condition is not respected, function will fail (@return an errorCode).
+ * The LZ4F_preferences_t structure is optional : you may provide NULL as argument,
+ * but it's not recommended, as it's the only way to provide dictID in the frame header.
+ * @return : number of bytes written into dstBuffer.
+ * or an error code if it fails (can be tested using LZ4F_isError()) */
+LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict(
+ LZ4F_cctx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const LZ4F_CDict* cdict,
+ const LZ4F_preferences_t* preferencesPtr);
+
+
+/*! LZ4F_compressBegin_usingCDict() :
+ * Inits streaming dictionary compression, and writes the frame header into dstBuffer.
+ * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
+ * `prefsPtr` is optional : you may provide NULL as argument,
+ * however, it's the only way to provide dictID in the frame header.
+ * @return : number of bytes written into dstBuffer for the header,
+ * or an error code (which can be tested using LZ4F_isError()) */
+LZ4FLIB_STATIC_API size_t LZ4F_compressBegin_usingCDict(
+ LZ4F_cctx* cctx,
+ void* dstBuffer, size_t dstCapacity,
+ const LZ4F_CDict* cdict,
+ const LZ4F_preferences_t* prefsPtr);
+
+
+/*! LZ4F_decompress_usingDict() :
+ * Same as LZ4F_decompress(), using a predefined dictionary.
+ * Dictionary is used "in place", without any preprocessing.
+ * It must remain accessible throughout the entire frame decoding. */
+LZ4FLIB_STATIC_API size_t LZ4F_decompress_usingDict(
+ LZ4F_dctx* dctxPtr,
+ void* dstBuffer, size_t* dstSizePtr,
+ const void* srcBuffer, size_t* srcSizePtr,
+ const void* dict, size_t dictSize,
+ const LZ4F_decompressOptions_t* decompressOptionsPtr);
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* defined(LZ4F_STATIC_LINKING_ONLY) && !defined(LZ4F_H_STATIC_09782039843) */
diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h
index be587e6..925a2c5 100644
--- a/lib/lz4frame_static.h
+++ b/lib/lz4frame_static.h
@@ -36,121 +36,12 @@
#ifndef LZ4FRAME_STATIC_H_0398209384
#define LZ4FRAME_STATIC_H_0398209384
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-/* lz4frame_static.h should be used solely in the context of static linking.
- * It contains definitions which are not stable and may change in the future.
- * Never use it in the context of DLL linking.
- *
- * Defining LZ4F_PUBLISH_STATIC_FUNCTIONS allows one to override this. Use at
- * your own risk.
+/* The declarations that formerly were made here have been merged into
+ * lz4frame.h, protected by the LZ4F_STATIC_LINKING_ONLY macro. Going forward,
+ * it is recommended to simply include that header directly.
*/
-#ifdef LZ4F_PUBLISH_STATIC_FUNCTIONS
-#define LZ4FLIB_STATIC_API LZ4FLIB_API
-#else
-#define LZ4FLIB_STATIC_API
-#endif
-
-/* --- Dependency --- */
+#define LZ4F_STATIC_LINKING_ONLY
#include "lz4frame.h"
-
-/* --- Error List --- */
-#define LZ4F_LIST_ERRORS(ITEM) \
- ITEM(OK_NoError) \
- ITEM(ERROR_GENERIC) \
- ITEM(ERROR_maxBlockSize_invalid) \
- ITEM(ERROR_blockMode_invalid) \
- ITEM(ERROR_contentChecksumFlag_invalid) \
- ITEM(ERROR_compressionLevel_invalid) \
- ITEM(ERROR_headerVersion_wrong) \
- ITEM(ERROR_blockChecksum_invalid) \
- ITEM(ERROR_reservedFlag_set) \
- ITEM(ERROR_allocation_failed) \
- ITEM(ERROR_srcSize_tooLarge) \
- ITEM(ERROR_dstMaxSize_tooSmall) \
- ITEM(ERROR_frameHeader_incomplete) \
- ITEM(ERROR_frameType_unknown) \
- ITEM(ERROR_frameSize_wrong) \
- ITEM(ERROR_srcPtr_wrong) \
- ITEM(ERROR_decompressionFailed) \
- ITEM(ERROR_headerChecksum_invalid) \
- ITEM(ERROR_contentChecksum_invalid) \
- ITEM(ERROR_frameDecoding_alreadyStarted) \
- ITEM(ERROR_maxCode)
-
-#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM,
-
-/* enum list is exposed, to handle specific errors */
-typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes;
-
-LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult);
-
-
-
-/**********************************
- * Bulk processing dictionary API
- *********************************/
-typedef struct LZ4F_CDict_s LZ4F_CDict;
-
-/*! LZ4_createCDict() :
- * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
- * LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
- * LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
- * `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict */
-LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize);
-LZ4FLIB_STATIC_API void LZ4F_freeCDict(LZ4F_CDict* CDict);
-
-
-/*! LZ4_compressFrame_usingCDict() :
- * Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary.
- * cctx must point to a context created by LZ4F_createCompressionContext().
- * If cdict==NULL, compress without a dictionary.
- * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
- * If this condition is not respected, function will fail (@return an errorCode).
- * The LZ4F_preferences_t structure is optional : you may provide NULL as argument,
- * but it's not recommended, as it's the only way to provide dictID in the frame header.
- * @return : number of bytes written into dstBuffer.
- * or an error code if it fails (can be tested using LZ4F_isError()) */
-LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict(
- LZ4F_cctx* cctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const LZ4F_CDict* cdict,
- const LZ4F_preferences_t* preferencesPtr);
-
-
-/*! LZ4F_compressBegin_usingCDict() :
- * Inits streaming dictionary compression, and writes the frame header into dstBuffer.
- * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
- * `prefsPtr` is optional : you may provide NULL as argument,
- * however, it's the only way to provide dictID in the frame header.
- * @return : number of bytes written into dstBuffer for the header,
- * or an error code (which can be tested using LZ4F_isError()) */
-LZ4FLIB_STATIC_API size_t LZ4F_compressBegin_usingCDict(
- LZ4F_cctx* cctx,
- void* dstBuffer, size_t dstCapacity,
- const LZ4F_CDict* cdict,
- const LZ4F_preferences_t* prefsPtr);
-
-
-/*! LZ4F_decompress_usingDict() :
- * Same as LZ4F_decompress(), using a predefined dictionary.
- * Dictionary is used "in place", without any preprocessing.
- * It must remain accessible throughout the entire frame decoding. */
-LZ4FLIB_STATIC_API size_t LZ4F_decompress_usingDict(
- LZ4F_dctx* dctxPtr,
- void* dstBuffer, size_t* dstSizePtr,
- const void* srcBuffer, size_t* srcSizePtr,
- const void* dict, size_t dictSize,
- const LZ4F_decompressOptions_t* decompressOptionsPtr);
-
-
-#if defined (__cplusplus)
-}
-#endif
-
#endif /* LZ4FRAME_STATIC_H_0398209384 */
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 111a09b..4126ef8 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -79,21 +79,33 @@
static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); }
+/*=== Enums ===*/
+typedef enum { noDictCtx, usingDictCtx } dictCtx_directive;
/**************************************
* HC Compression
**************************************/
-static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
+static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4)
{
MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
- hc4->nextToUpdate = 64 KB;
- hc4->base = start - 64 KB;
+}
+
+static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
+{
+ uptrval startingOffset = hc4->end - hc4->base;
+ if (startingOffset > 1 GB) {
+ LZ4HC_clearTables(hc4);
+ startingOffset = 0;
+ }
+ startingOffset += 64 KB;
+ hc4->nextToUpdate = (U32) startingOffset;
+ hc4->base = start - startingOffset;
hc4->end = start;
- hc4->dictBase = start - 64 KB;
- hc4->dictLimit = 64 KB;
- hc4->lowLimit = 64 KB;
+ hc4->dictBase = start - startingOffset;
+ hc4->dictLimit = (U32) startingOffset;
+ hc4->lowLimit = (U32) startingOffset;
}
@@ -198,19 +210,23 @@ LZ4HC_InsertAndGetWiderMatch (
const BYTE** matchpos,
const BYTE** startpos,
const int maxNbAttempts,
- const int patternAnalysis)
+ const int patternAnalysis,
+ const dictCtx_directive dict)
{
U16* const chainTable = hc4->chainTable;
U32* const HashTable = hc4->hashTable;
+ const LZ4HC_CCtx_internal * const dictCtx = hc4->dictCtx;
const BYTE* const base = hc4->base;
const U32 dictLimit = hc4->dictLimit;
const BYTE* const lowPrefixPtr = base + dictLimit;
- const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - MAX_DISTANCE;
+ const U32 ipIndex = (U32)(ip - base);
+ const U32 lowLimit = (hc4->lowLimit + 64 KB > ipIndex) ? hc4->lowLimit : ipIndex - MAX_DISTANCE;
const BYTE* const dictBase = hc4->dictBase;
int const delta = (int)(ip-iLowLimit);
int nbAttempts = maxNbAttempts;
U32 const pattern = LZ4_read32(ip);
U32 matchIndex;
+ U32 dictMatchIndex;
repeat_state_e repeat = rep_untested;
size_t srcPatternLength = 0;
@@ -287,6 +303,37 @@ LZ4HC_InsertAndGetWiderMatch (
} } } }
} /* while ((matchIndex>=lowLimit) && (nbAttempts)) */
+ if (dict == usingDictCtx && nbAttempts && ipIndex - lowLimit < MAX_DISTANCE) {
+ size_t const dictEndOffset = dictCtx->end - dictCtx->base;
+ assert(dictEndOffset <= 1 GB);
+ dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
+ matchIndex = dictMatchIndex + lowLimit - (U32)dictEndOffset;
+ while (ipIndex - matchIndex <= MAX_DISTANCE && nbAttempts--) {
+ const BYTE* const matchPtr = dictCtx->base + dictMatchIndex;
+
+ if (LZ4_read32(matchPtr) == pattern) {
+ int mlt;
+ int back = 0;
+ const BYTE* vLimit = ip + (dictEndOffset - dictMatchIndex);
+ if (vLimit > iHighLimit) vLimit = iHighLimit;
+ mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
+ back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0;
+ mlt -= back;
+ if (mlt > longest) {
+ longest = mlt;
+ *matchpos = base + matchIndex + back;
+ *startpos = ip + back;
+ }
+ }
+
+ {
+ U32 const nextOffset = DELTANEXTU16(dictCtx->chainTable, dictMatchIndex);
+ dictMatchIndex -= nextOffset;
+ matchIndex -= nextOffset;
+ }
+ }
+ }
+
return longest;
}
@@ -295,13 +342,14 @@ int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index tabl
const BYTE* const ip, const BYTE* const iLimit,
const BYTE** matchpos,
const int maxNbAttempts,
- const int patternAnalysis)
+ const int patternAnalysis,
+ const dictCtx_directive dict)
{
const BYTE* uselessPtr = ip;
/* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
* but this won't be the case here, as we define iLowLimit==ip,
* so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
- return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis);
+ return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis, dict);
}
@@ -384,14 +432,15 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
return 0;
}
-static int LZ4HC_compress_hashChain (
+LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
LZ4HC_CCtx_internal* const ctx,
const char* const source,
char* const dest,
int* srcSizePtr,
int const maxOutputSize,
unsigned maxNbAttempts,
- limitedOutput_directive limit
+ const limitedOutput_directive limit,
+ const dictCtx_directive dict
)
{
const int inputSize = *srcSizePtr;
@@ -423,7 +472,7 @@ static int LZ4HC_compress_hashChain (
/* Main Loop */
while (ip <= mflimit) {
- ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis);
+ ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis, dict);
if (ml<MINMATCH) { ip++; continue; }
/* saved, in case we would skip too much */
@@ -435,7 +484,7 @@ _Search2:
if (ip+ml <= mflimit)
ml2 = LZ4HC_InsertAndGetWiderMatch(ctx,
ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2,
- maxNbAttempts, patternAnalysis);
+ maxNbAttempts, patternAnalysis, dict);
else
ml2 = ml;
@@ -482,7 +531,7 @@ _Search3:
if (start2 + ml2 <= mflimit)
ml3 = LZ4HC_InsertAndGetWiderMatch(ctx,
start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3,
- maxNbAttempts, patternAnalysis);
+ maxNbAttempts, patternAnalysis, dict);
else
ml3 = ml2;
@@ -606,17 +655,19 @@ static int LZ4HC_compress_optimal( LZ4HC_CCtx_internal* ctx,
const char* const source, char* dst,
int* srcSizePtr, int dstCapacity,
int const nbSearches, size_t sufficient_len,
- limitedOutput_directive limit, int const fullUpdate);
+ const limitedOutput_directive limit, int const fullUpdate,
+ const dictCtx_directive dict);
-static int LZ4HC_compress_generic (
+LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
LZ4HC_CCtx_internal* const ctx,
const char* const src,
char* const dst,
int* const srcSizePtr,
int const dstCapacity,
int cLevel,
- limitedOutput_directive limit
+ const limitedOutput_directive limit,
+ const dictCtx_directive dict
)
{
typedef enum { lz4hc, lz4opt } lz4hc_strat_e;
@@ -641,6 +692,8 @@ static int LZ4HC_compress_generic (
{ lz4opt,8192, LZ4_OPT_NUM }, /* 12==LZ4HC_CLEVEL_MAX */
};
+ DEBUGLOG(4, "LZ4HC_compress_generic(%p, %p, %d)", ctx, src, *srcSizePtr);
+
if (limit == limitedDestSize && dstCapacity < 1) return 0; /* Impossible to store anything */
if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size (too large or negative) */
@@ -653,22 +706,81 @@ static int LZ4HC_compress_generic (
if (cParam.strat == lz4hc)
return LZ4HC_compress_hashChain(ctx,
src, dst, srcSizePtr, dstCapacity,
- cParam.nbSearches, limit);
+ cParam.nbSearches, limit, dict);
assert(cParam.strat == lz4opt);
return LZ4HC_compress_optimal(ctx,
src, dst, srcSizePtr, dstCapacity,
cParam.nbSearches, cParam.targetLength, limit,
- cLevel == LZ4HC_CLEVEL_MAX); /* ultra mode */
+ cLevel == LZ4HC_CLEVEL_MAX, dict); /* ultra mode */
+ }
+}
+
+static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock);
+
+static int LZ4HC_compress_generic_noDictCtx (
+ LZ4HC_CCtx_internal* const ctx,
+ const char* const src,
+ char* const dst,
+ int* const srcSizePtr,
+ int const dstCapacity,
+ int cLevel,
+ limitedOutput_directive limit
+ )
+{
+ assert(ctx->dictCtx == NULL);
+ return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, noDictCtx);
+}
+
+static int LZ4HC_compress_generic_dictCtx (
+ LZ4HC_CCtx_internal* const ctx,
+ const char* const src,
+ char* const dst,
+ int* const srcSizePtr,
+ int const dstCapacity,
+ int cLevel,
+ limitedOutput_directive limit
+ )
+{
+ const size_t position = ctx->end - ctx->base - ctx->lowLimit;
+ assert(ctx->dictCtx != NULL);
+ if (position >= 64 KB) {
+ ctx->dictCtx = NULL;
+ return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
+ } else if (position == 0 && *srcSizePtr > 4 KB) {
+ memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal));
+ LZ4HC_setExternalDict(ctx, (const BYTE *)src);
+ ctx->compressionLevel = cLevel;
+ return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
+ } else {
+ return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx);
+ }
+}
+
+static int LZ4HC_compress_generic (
+ LZ4HC_CCtx_internal* const ctx,
+ const char* const src,
+ char* const dst,
+ int* const srcSizePtr,
+ int const dstCapacity,
+ int cLevel,
+ limitedOutput_directive limit
+ )
+{
+ if (ctx->dictCtx == NULL) {
+ return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
+ } else {
+ return LZ4HC_compress_generic_dictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
}
}
int LZ4_sizeofStateHC(void) { return sizeof(LZ4_streamHC_t); }
-int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
+int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
{
LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */
+ LZ4_resetStreamHC_fast((LZ4_streamHC_t*)state, compressionLevel);
LZ4HC_init (ctx, (const BYTE*)src);
if (dstCapacity < LZ4_compressBound(srcSize))
return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, limitedOutput);
@@ -676,6 +788,13 @@ int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int src
return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, noLimit);
}
+int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
+{
+ if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */
+ LZ4_resetStreamHC ((LZ4_streamHC_t*)state, compressionLevel);
+ return LZ4_compress_HC_extStateHC_fastReset(state, src, dst, srcSize, dstCapacity, compressionLevel);
+}
+
int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
{
#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
@@ -696,6 +815,7 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in
int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int cLevel)
{
LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse;
+ LZ4_resetStreamHC((LZ4_streamHC_t*)LZ4HC_Data, cLevel);
LZ4HC_init(ctx, (const BYTE*) source);
return LZ4HC_compress_generic(ctx, source, dest, sourceSizePtr, targetDestSize, cLevel, limitedDestSize);
}
@@ -706,8 +826,16 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i
* Streaming Functions
**************************************/
/* allocation */
-LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); }
-int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) {
+LZ4_streamHC_t* LZ4_createStreamHC(void) {
+ LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
+ LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
+ LZ4_streamHCPtr->internal_donotuse.base = NULL;
+ LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
+ return LZ4_streamHCPtr;
+}
+
+int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) {
+ DEBUGLOG(4, "LZ4_freeStreamHC(%p)", LZ4_streamHCPtr);
if (!LZ4_streamHCPtr) return 0; /* support free on NULL */
free(LZ4_streamHCPtr);
return 0;
@@ -718,7 +846,19 @@ int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) {
void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
{
LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
+ DEBUGLOG(4, "LZ4_resetStreamHC(%p, %d)", LZ4_streamHCPtr, compressionLevel);
+ LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
+ LZ4_streamHCPtr->internal_donotuse.base = NULL;
+ LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
+ LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
+}
+
+void LZ4_resetStreamHC_fast (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
+{
+ DEBUGLOG(4, "LZ4_resetStreamHC_fast(%p, %d)", LZ4_streamHCPtr, compressionLevel);
+ LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base;
LZ4_streamHCPtr->internal_donotuse.base = NULL;
+ LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
}
@@ -732,21 +872,27 @@ void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLev
int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize)
{
LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
+ DEBUGLOG(4, "LZ4_loadDictHC(%p, %p, %d)", LZ4_streamHCPtr, dictionary, dictSize);
if (dictSize > 64 KB) {
dictionary += dictSize - 64 KB;
dictSize = 64 KB;
}
LZ4HC_init (ctxPtr, (const BYTE*)dictionary);
+ LZ4HC_clearTables (ctxPtr);
ctxPtr->end = (const BYTE*)dictionary + dictSize;
if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);
return dictSize;
}
+void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC_t *dictionary_stream) {
+ working_stream->internal_donotuse.dictCtx = dictionary_stream != NULL ? &(dictionary_stream->internal_donotuse) : NULL;
+}
/* compression */
static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock)
{
+ DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock);
if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */
/* Only one memory segment for extDict, so any previous extDict is lost at this stage */
@@ -764,6 +910,7 @@ static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr,
limitedOutput_directive limit)
{
LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
+ DEBUGLOG(4, "LZ4_compressHC_continue_generic(%p, %p, %d)", LZ4_streamHCPtr, src, *srcSizePtr);
/* auto-init if forgotten */
if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) src);
@@ -812,6 +959,7 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS
{
LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse;
int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));
+ DEBUGLOG(4, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize);
if (dictSize > 64 KB) dictSize = 64 KB;
if (dictSize < 4) dictSize = 0;
if (dictSize > prefixSize) dictSize = prefixSize;
@@ -851,8 +999,8 @@ int LZ4_resetStreamStateHC(void* state, char* inputBuffer)
{
LZ4HC_CCtx_internal *ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */
+ LZ4_resetStreamHC((LZ4_streamHC_t*)state, ((LZ4_streamHC_t*)state)->internal_donotuse.compressionLevel);
LZ4HC_init(ctx, (const BYTE*)inputBuffer);
- ctx->inputBuffer = inputBuffer;
return 0;
}
@@ -860,9 +1008,8 @@ void* LZ4_createHC (const char* inputBuffer)
{
LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t));
if (hc4 == NULL) return NULL; /* not enough memory */
+ LZ4_resetStreamHC(hc4, 0 /* compressionLevel */);
LZ4HC_init (&hc4->internal_donotuse, (const BYTE*)inputBuffer);
- assert(sizeof(size_t) == sizeof(void*));
- hc4->internal_donotuse.inputBuffer = (void*)(size_t)inputBuffer; /* ugly hack, circumvent -Wcast-qual */
return hc4;
}
@@ -884,9 +1031,11 @@ int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* src, c
char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
{
- LZ4HC_CCtx_internal* const hc4 = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse;
- int const dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB);
- return (char*)(hc4->inputBuffer) + dictSize;
+ LZ4_streamHC_t *ctx = (LZ4_streamHC_t*)LZ4HC_Data;
+ const BYTE *bufferStart = ctx->internal_donotuse.base + ctx->internal_donotuse.lowLimit;
+ LZ4_resetStreamHC_fast(ctx, ctx->internal_donotuse.compressionLevel);
+ /* avoid const char * -> char * conversion warning :( */
+ return (char *)(uptrval)bufferStart;
}
@@ -932,7 +1081,8 @@ typedef struct {
LZ4_FORCE_INLINE LZ4HC_match_t
LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
const BYTE* ip, const BYTE* const iHighLimit,
- int minLen, int nbSearches)
+ int minLen, int nbSearches,
+ const dictCtx_directive dict)
{
LZ4HC_match_t match = { 0 , 0 };
const BYTE* matchPtr = NULL;
@@ -941,7 +1091,7 @@ LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
* so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
int const matchLength = LZ4HC_InsertAndGetWiderMatch(ctx,
ip, ip, iHighLimit, minLen, &matchPtr, &ip,
- nbSearches, 1 /* patternAnalysis */);
+ nbSearches, 1 /* patternAnalysis */, dict);
if (matchLength <= minLen) return match;
match.len = matchLength;
match.off = (int)(ip-matchPtr);
@@ -957,8 +1107,9 @@ static int LZ4HC_compress_optimal (
int dstCapacity,
int const nbSearches,
size_t sufficient_len,
- limitedOutput_directive limit,
- int const fullUpdate
+ const limitedOutput_directive limit,
+ int const fullUpdate,
+ const dictCtx_directive dict
)
{
#define TRAILING_LITERALS 3
@@ -986,7 +1137,7 @@ static int LZ4HC_compress_optimal (
int best_mlen, best_off;
int cur, last_match_pos = 0;
- LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches);
+ LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches, dict);
if (firstMatch.len==0) { ip++; continue; }
if ((size_t)firstMatch.len > sufficient_len) {
@@ -1056,10 +1207,10 @@ static int LZ4HC_compress_optimal (
DEBUGLOG(7, "search at rPos:%u", cur);
if (fullUpdate)
- newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches);
+ newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches, dict);
else
/* only test matches of minimum length; slightly faster, but misses a few bytes */
- newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches);
+ newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches, dict);
if (!newMatch.len) continue;
if ( ((size_t)newMatch.len > sufficient_len)
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 7a25bee..28e2528 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -141,35 +141,37 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in
#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
#include <stdint.h>
-typedef struct
+typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal;
+struct LZ4HC_CCtx_internal
{
uint32_t hashTable[LZ4HC_HASHTABLESIZE];
uint16_t chainTable[LZ4HC_MAXD];
const uint8_t* end; /* next block here to continue on current prefix */
const uint8_t* base; /* All index relative to this position */
const uint8_t* dictBase; /* alternate base for extDict */
- void* inputBuffer; /* deprecated */
uint32_t dictLimit; /* below that point, need extDict */
uint32_t lowLimit; /* below that point, no more dict */
uint32_t nextToUpdate; /* index from which to continue dictionary update */
int compressionLevel;
-} LZ4HC_CCtx_internal;
+ const LZ4HC_CCtx_internal* dictCtx;
+};
#else
-typedef struct
+typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal;
+struct LZ4HC_CCtx_internal
{
unsigned int hashTable[LZ4HC_HASHTABLESIZE];
unsigned short chainTable[LZ4HC_MAXD];
const unsigned char* end; /* next block here to continue on current prefix */
const unsigned char* base; /* All index relative to this position */
const unsigned char* dictBase; /* alternate base for extDict */
- void* inputBuffer; /* deprecated */
unsigned int dictLimit; /* below that point, need extDict */
unsigned int lowLimit; /* below that point, no more dict */
unsigned int nextToUpdate; /* index from which to continue dictionary update */
int compressionLevel;
-} LZ4HC_CCtx_internal;
+ const LZ4HC_CCtx_internal* dictCtx;
+};
#endif
@@ -206,7 +208,14 @@ LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_co
LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
-/* Deprecated Streaming functions; should no longer be used */
+/* Obsolete streaming functions; degraded functionality; do not use!
+ *
+ * In order to perform streaming compression, these functions depended on data
+ * that is no longer tracked in the state. They have been preserved as well as
+ * possible: using them will still produce a correct output. However, use of
+ * LZ4_slideInputBufferHC() will truncate the history of the stream, rather
+ * than preserve a window-sized chunk of history.
+ */
LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (const char* inputBuffer);
LZ4_DEPRECATED("use LZ4_saveDictHC() instead") LZ4LIB_API char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") LZ4LIB_API int LZ4_freeHC (void* LZ4HC_Data);
@@ -266,7 +275,65 @@ int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr,
*/
void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
-
+/*! LZ4_resetStreamHC_fast() :
+ * When an LZ4_streamHC_t is known to be in a internally coherent state,
+ * it can often be prepared for a new compression with almost no work, only
+ * sometimes falling back to the full, expensive reset that is always required
+ * when the stream is in an indeterminate state (i.e., the reset performed by
+ * LZ4_resetStreamHC()).
+ *
+ * LZ4_streamHCs are guaranteed to be in a valid state when:
+ * - returned from LZ4_createStreamHC()
+ * - reset by LZ4_resetStreamHC()
+ * - memset(stream, 0, sizeof(LZ4_streamHC_t))
+ * - the stream was in a valid state and was reset by LZ4_resetStreamHC_fast()
+ * - the stream was in a valid state and was then used in any compression call
+ * that returned success
+ * - the stream was in an indeterminate state and was used in a compression
+ * call that fully reset the state (LZ4_compress_HC_extStateHC()) and that
+ * returned success
+ */
+void LZ4_resetStreamHC_fast(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
+
+/*! LZ4_compress_HC_extStateHC_fastReset() :
+ * A variant of LZ4_compress_HC_extStateHC().
+ *
+ * Using this variant avoids an expensive initialization step. It is only safe
+ * to call if the state buffer is known to be correctly initialized already
+ * (see above comment on LZ4_resetStreamHC_fast() for a definition of
+ * "correctly initialized"). From a high level, the difference is that this
+ * function initializes the provided state with a call to
+ * LZ4_resetStreamHC_fast() while LZ4_compress_HC_extStateHC() starts with a
+ * call to LZ4_resetStreamHC().
+ */
+int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel);
+
+/*! LZ4_attach_HC_dictionary() :
+ * This is an experimental API that allows for the efficient use of a
+ * static dictionary many times.
+ *
+ * Rather than re-loading the dictionary buffer into a working context before
+ * each compression, or copying a pre-loaded dictionary's LZ4_streamHC_t into a
+ * working LZ4_streamHC_t, this function introduces a no-copy setup mechanism,
+ * in which the working stream references the dictionary stream in-place.
+ *
+ * Several assumptions are made about the state of the dictionary stream.
+ * Currently, only streams which have been prepared by LZ4_loadDictHC() should
+ * be expected to work.
+ *
+ * Alternatively, the provided dictionary stream pointer may be NULL, in which
+ * case any existing dictionary stream is unset.
+ *
+ * A dictionary should only be attached to a stream without any history (i.e.,
+ * a stream that has just been reset).
+ *
+ * The dictionary will remain attached to the working stream only for the
+ * current stream session. Calls to LZ4_resetStreamHC(_fast) will remove the
+ * dictionary context association from the working stream. The dictionary
+ * stream (and source buffer) must remain in-place / accessible / unchanged
+ * through the lifetime of the stream session.
+ */
+LZ4LIB_API void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC_t *dictionary_stream);
#endif /* LZ4_HC_SLO_098092834 */
#endif /* LZ4_HC_STATIC_LINKING_ONLY */
diff --git a/programs/lz4io.c b/programs/lz4io.c
index 6d0d0d0..ccf4fa1 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -56,8 +56,8 @@
#include "lz4io.h"
#include "lz4.h" /* still required for legacy format */
#include "lz4hc.h" /* still required for legacy format */
+#define LZ4F_STATIC_LINKING_ONLY
#include "lz4frame.h"
-#include "lz4frame_static.h"
/*****************************
diff --git a/programs/util.h b/programs/util.h
index ff25106..ef6ca77 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -30,8 +30,9 @@ extern "C" {
* Dependencies
******************************************/
#include "platform.h" /* PLATFORM_POSIX_VERSION */
-#include <stdlib.h> /* malloc */
#include <stddef.h> /* size_t, ptrdiff_t */
+#include <stdlib.h> /* malloc */
+#include <string.h> /* strlen, strncpy */
#include <stdio.h> /* fprintf */
#include <sys/types.h> /* stat, utime */
#include <sys/stat.h> /* stat */
diff --git a/tests/Makefile b/tests/Makefile
index 2b93c9f..d4847b1 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -80,7 +80,8 @@ lz4c32: # create a 32-bits version for 32/64 interop tests
%.o : $(LZ4DIR)/%.c $(LZ4DIR)/%.h
$(CC) -c $(CFLAGS) $(CPPFLAGS) $< -o $@
-fullbench : lz4.o lz4hc.o lz4frame.o xxhash.o fullbench.c
+fullbench : DEBUGLEVEL=0
+fullbench : lz4.o lz4hc.o lz4frame.o xxhash.o fullbench.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
$(LZ4DIR)/liblz4.a:
diff --git a/tests/frametest.c b/tests/frametest.c
index 74d9c88..7d69ff7 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -41,7 +41,11 @@
#include <string.h> /* strcmp */
#include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */
#include <assert.h>
-#include "lz4frame_static.h"
+#include "lz4frame.h" /* include multiple times to test correctness/safety */
+#include "lz4frame.h"
+#define LZ4F_STATIC_LINKING_ONLY
+#include "lz4frame.h"
+#include "lz4frame.h"
#include "lz4.h" /* LZ4_VERSION_STRING */
#define XXH_STATIC_LINKING_ONLY
#include "xxhash.h" /* XXH64 */
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 244cc4f..8cd4dec 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -446,7 +446,12 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
/* Test compression HC using external state */
FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC()");
ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
- FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed");
+ FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed")
+
+ /* Test compression HC using fast reset external state */
+ FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC_fastReset()");
+ ret = LZ4_compress_HC_extStateHC_fastReset(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
+ FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC_fastReset() failed");
/* Test compression using external state */
FUZ_DISPLAYTEST("test LZ4_compress_fast_extState()");
@@ -487,6 +492,31 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");
+ /* Test decoding with empty input */
+ FUZ_DISPLAYTEST("LZ4_decompress_safe() with empty input");
+ LZ4_decompress_safe(NULL, decodedBuffer, 0, blockSize);
+
+ /* Test decoding with a one byte input */
+ FUZ_DISPLAYTEST("LZ4_decompress_safe() with one byte input");
+ { char const tmp = 0xFF;
+ LZ4_decompress_safe(&tmp, decodedBuffer, 1, blockSize);
+ }
+
+ /* Test decoding shortcut edge case */
+ FUZ_DISPLAYTEST("LZ4_decompress_safe() with shortcut edge case");
+ { char tmp[17];
+ /* 14 bytes of literals, followed by a 14 byte match.
+ * Should not read beyond the end of the buffer.
+ * See https://github.com/lz4/lz4/issues/508. */
+ *tmp = 0xEE;
+ memset(tmp + 1, 0, 14);
+ tmp[15] = 14;
+ tmp[16] = 0;
+ ret = LZ4_decompress_safe(tmp, decodedBuffer, sizeof(tmp), blockSize);
+ FUZ_CHECKTEST(ret >= 0, "LZ4_decompress_safe() should fail");
+ }
+
+
/* Test decoding with output size exactly what's necessary => must work */
FUZ_DISPLAYTEST();
decodedBuffer[blockSize] = 0;
@@ -810,6 +840,49 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
}
+ /* Compress HC using external dictionary stream */
+ FUZ_DISPLAYTEST();
+ {
+ LZ4_streamHC_t LZ4_streamHC;
+
+ LZ4_resetStreamHC (&LZ4dictHC, compressionLevel);
+ LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
+ LZ4_resetStreamHC (&LZ4_streamHC, compressionLevel);
+ LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC);
+ blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, (int)compressedBufferSize);
+ FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue with ExtDictCtx failed");
+
+ FUZ_DISPLAYTEST();
+ LZ4_resetStreamHC (&LZ4_streamHC, compressionLevel);
+ LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC);
+ ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
+ FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDictCtx should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize);
+
+ FUZ_DISPLAYTEST();
+ LZ4_resetStreamHC (&LZ4_streamHC, compressionLevel);
+ LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC);
+ ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
+ FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx size is different (%i != %i)", ret, blockContinueCompressedSize);
+ FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue using ExtDictCtx should work : enough size available within output buffer");
+
+ FUZ_DISPLAYTEST();
+ LZ4_resetStreamHC_fast (&LZ4_streamHC, compressionLevel);
+ LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC);
+ ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
+ FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx and fast reset size is different (%i != %i)", ret, blockContinueCompressedSize);
+ FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue using ExtDictCtx and fast reset should work : enough size available within output buffer");
+
+ FUZ_DISPLAYTEST();
+ decodedBuffer[blockSize] = 0;
+ ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
+ FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
+ FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
+ { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+ if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
+ FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+ }
+ }
+
/* Compress HC continue destSize */
FUZ_DISPLAYTEST();
{ int const availableSpace = (FUZ_rand(&randState) % blockSize) + 5;
generated by cgit v0.12 at 2025-01-09 11:07:37 (GMT)