summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorYann Collet <yann.collet.73@gmail.com>2014-10-18 10:18:14 (GMT)
committerYann Collet <yann.collet.73@gmail.com>2014-10-18 10:18:14 (GMT)
commitd239a23337e5eba41f557b48eb26f0db81b28f26 (patch)
tree63d7ea0dc4d6224656392c5772053e5bf4fbd3a0
parent460616007c032c7a9a6e0591c97bf025689cc001 (diff)
downloadlz4-d239a23337e5eba41f557b48eb26f0db81b28f26.zip
lz4-d239a23337e5eba41f557b48eb26f0db81b28f26.tar.gz
lz4-d239a23337e5eba41f557b48eb26f0db81b28f26.tar.bz2
updated LZ4HC API
-rw-r--r--lz4.h10
-rw-r--r--lz4hc.c368
-rw-r--r--lz4hc.h58
-rw-r--r--programs/fullbench.c4
-rw-r--r--programs/fuzzer.c90
5 files changed, 333 insertions, 197 deletions
diff --git a/lz4.h b/lz4.h
index 44ada14..7ad736f 100644
--- a/lz4.h
+++ b/lz4.h
@@ -193,7 +193,7 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_streamPtr);
* LZ4_freeStream releases its memory.
*/
LZ4_stream_t* LZ4_createStream(void);
-int LZ4_freeStream (LZ4_stream_t* LZ4_stream);
+int LZ4_freeStream (LZ4_stream_t* LZ4_streamPtr);
/*
* LZ4_loadDict
@@ -202,21 +202,21 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_stream);
* Loading a size of 0 is allowed.
* Return : 1 if OK, 0 if error
*/
-int LZ4_loadDict (LZ4_stream_t* LZ4_stream, const char* dictionary, int dictSize);
+int LZ4_loadDict (LZ4_stream_t* LZ4_streamPtr, const char* dictionary, int dictSize);
/*
* LZ4_compress_continue
* Compress data block 'source', using blocks compressed before as dictionary to improve compression ratio
* Previous data blocks are assumed to still be present at their previous location.
*/
-int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize);
+int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
/*
* LZ4_compress_limitedOutput_continue
* Same as before, but also specify a maximum target compressed size (maxOutputSize)
* If objective cannot be met, compression exits, and returns a zero.
*/
-int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize);
+int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
/*
* LZ4_saveDict
@@ -227,7 +227,7 @@ int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* s
* Return : dictionary size in bytes, or 0 if error
* Note : any dictSize > 64 KB will be interpreted as 64KB.
*/
-int LZ4_saveDict (LZ4_stream_t* LZ4_stream, char* safeBuffer, int dictSize);
+int LZ4_saveDict (LZ4_stream_t* LZ4_streamPtr, char* safeBuffer, int dictSize);
/************************************************
diff --git a/lz4hc.c b/lz4hc.c
index 34a6173..2cf494e 100644
--- a/lz4hc.c
+++ b/lz4hc.c
@@ -214,8 +214,8 @@ typedef struct _U64_S { U64 v; } _PACKED U64_S;
#define MINLENGTH (MFLIMIT+1)
#define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH)
-#define KB *(1U<<10)
-#define MB *(1U<<20)
+#define KB *(1<<10)
+#define MB *(1<<20)
#define GB *(1U<<30)
@@ -226,16 +226,12 @@ typedef struct _U64_S { U64 v; } _PACKED U64_S;
# define STEPSIZE 8
# define LZ4_COPYSTEP(s,d) A64(d) = A64(s); d+=8; s+=8;
# define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d)
-# define AARCH A64
-# define HTYPE U32
-# define INITBASE(b,s) const BYTE* const b = s
+# define AARCH A64
#else /* 32-bit */
# define STEPSIZE 4
# define LZ4_COPYSTEP(s,d) A32(d) = A32(s); d+=4; s+=4;
# define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d); LZ4_COPYSTEP(s,d);
-# define AARCH A32
-# define HTYPE U32
-# define INITBASE(b,s) const BYTE* const b = s
+# define AARCH A32
#endif
#if defined(LZ4_BIG_ENDIAN)
@@ -252,18 +248,20 @@ typedef struct _U64_S { U64 v; } _PACKED U64_S;
**************************************/
typedef struct
{
+ U32 hashTable[HASHTABLESIZE];
+ U16 chainTable[MAXD];
const BYTE* inputBuffer;
const BYTE* base;
const BYTE* end;
- HTYPE hashTable[HASHTABLESIZE];
- U16 chainTable[MAXD];
- const BYTE* nextToUpdate;
+ U32 nextToUpdate;
+ U32 compressionLevel;
} LZ4HC_Data_Structure;
/**************************************
Macros
**************************************/
+#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(!!(c)) }; } /* Visual : use only *after* variable declarations */
#define LZ4_WILDCOPY(s,d,e) do { LZ4_COPYPACKET(s,d) } while (d<e);
#define LZ4_BLINDCOPY(s,d,l) { BYTE* e=d+l; LZ4_WILDCOPY(s,d,e); d=e; }
#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG))
@@ -342,87 +340,41 @@ FORCE_INLINE int LZ4_NbCommonBytes (register U32 val)
#endif
-int LZ4_sizeofStreamStateHC()
-{
- return sizeof(LZ4HC_Data_Structure);
-}
-
-FORCE_INLINE void LZ4_initHC (LZ4HC_Data_Structure* hc4, const BYTE* base)
+FORCE_INLINE void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* base)
{
MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
- hc4->nextToUpdate = base + 1;
+ hc4->nextToUpdate = 1;
hc4->base = base;
hc4->inputBuffer = base;
hc4->end = base;
}
-int LZ4_resetStreamStateHC(void* state, const char* inputBuffer)
-{
- if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */
- LZ4_initHC((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer);
- return 0;
-}
-
-
-void* LZ4_createHC (const char* inputBuffer)
-{
- void* hc4 = ALLOCATOR(sizeof(LZ4HC_Data_Structure));
- LZ4_initHC ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer);
- return hc4;
-}
-
-
-int LZ4_freeHC (void* LZ4HC_Data)
-{
- FREEMEM(LZ4HC_Data);
- return (0);
-}
-
/* Update chains up to ip (excluded) */
FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip)
{
- U16* chainTable = hc4->chainTable;
- HTYPE* HashTable = hc4->hashTable;
- INITBASE(base,hc4->base);
+ U16* chainTable = hc4->chainTable;
+ U32* HashTable = hc4->hashTable;
+ const BYTE* const base = hc4->base;
+ const U32 target = ip - base;
+ U32 idx = hc4->nextToUpdate;
- while(hc4->nextToUpdate < ip)
+ while(idx < target)
{
- const BYTE* const p = hc4->nextToUpdate;
- size_t delta = (p) - HASH_POINTER(p);
+ U32 h = HASH_VALUE(base+idx);
+ size_t delta = idx - HashTable[h];
if (delta>MAX_DISTANCE) delta = MAX_DISTANCE;
- DELTANEXT(p) = (U16)delta;
- HashTable[HASH_VALUE(p)] = (HTYPE)((p) - base);
- hc4->nextToUpdate++;
+ chainTable[idx & 0xFFFF] = (U16)delta;
+ HashTable[h] = idx;
+ idx++;
}
-}
-
-
-char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
-{
- LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data;
- size_t distance = (hc4->end - 64 KB) - hc4->inputBuffer;
-
- if (hc4->end <= hc4->inputBuffer + 64 KB) return (char*)(hc4->end); /* no update : less than 64KB within buffer */
- distance = (distance >> 16) << 16; /* Must be a multiple of 64 KB */
- LZ4HC_Insert(hc4, hc4->end - MINMATCH);
- memcpy((void*)(hc4->end - 64 KB - distance), (const void*)(hc4->end - 64 KB), 64 KB);
- hc4->nextToUpdate -= distance;
- hc4->base -= distance;
- if ((U32)(hc4->inputBuffer - hc4->base) > 1 GB + 64 KB) /* Avoid overflow */
- {
- int i;
- hc4->base += 1 GB;
- for (i=0; i<HASHTABLESIZE; i++) hc4->hashTable[i] -= 1 GB;
- }
- hc4->end -= distance;
- return (char*)(hc4->end);
+ hc4->nextToUpdate = target;
}
-FORCE_INLINE size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* const matchlimit)
+static size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* const matchlimit)
{
const BYTE* p1t = p1;
@@ -443,124 +395,79 @@ FORCE_INLINE size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BY
FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* const matchlimit, const BYTE** matchpos, const int maxNbAttempts)
{
U16* const chainTable = hc4->chainTable;
- HTYPE* const HashTable = hc4->hashTable;
- const BYTE* ref;
- INITBASE(base,hc4->base);
+ U32* const HashTable = hc4->hashTable;
+ const BYTE* const base = hc4->base;
+ int matchIndex;
+ const int idxLow = (ip-base) > 64 KB ? (ip - base) - 64 KB : 0;
+ const BYTE* match;
int nbAttempts=maxNbAttempts;
- size_t repl=0, ml=0;
- U16 delta=0; /* useless assignment, to remove an uninitialization warning */
+ size_t ml=0;
/* HC4 match finder */
LZ4HC_Insert(hc4, ip);
- ref = HASH_POINTER(ip);
-
-#define REPEAT_OPTIMIZATION
-#ifdef REPEAT_OPTIMIZATION
- /* Detect repetitive sequences of length <= 4 */
- if ((U32)(ip-ref) <= 4) /* potential repetition */
- {
- if (A32(ref) == A32(ip)) /* confirmed */
- {
- delta = (U16)(ip-ref);
- repl = ml = LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit) + MINMATCH;
- *matchpos = ref;
- }
- ref = GETNEXT(ref);
- }
-#endif
+ matchIndex = HashTable[HASH_VALUE(ip)];
- while (((U32)(ip-ref) <= MAX_DISTANCE) && (nbAttempts))
+ while ((matchIndex>idxLow) && (nbAttempts))
{
nbAttempts--;
- if (*(ref+ml) == *(ip+ml))
- if (A32(ref) == A32(ip))
- {
- size_t mlt = LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit) + MINMATCH;
- if (mlt > ml) { ml = mlt; *matchpos = ref; }
- }
- ref = GETNEXT(ref);
- }
-
-#ifdef REPEAT_OPTIMIZATION
- /* Complete table */
- if (repl)
- {
- const BYTE* ptr = ip;
- const BYTE* end;
-
- end = ip + repl - (MINMATCH-1);
- while(ptr < end-delta)
+ match = base + matchIndex;
+ if (*(match+ml) == *(ip+ml))
+ if (A32(match) == A32(ip))
{
- DELTANEXT(ptr) = delta; /* Pre-Load */
- ptr++;
+ size_t mlt = LZ4HC_CommonLength(ip+MINMATCH, match+MINMATCH, matchlimit) + MINMATCH;
+ if (mlt > ml) { ml = mlt; *matchpos = match; }
}
- do
- {
- DELTANEXT(ptr) = delta;
- HashTable[HASH_VALUE(ptr)] = (HTYPE)((ptr) - base); /* Head of chain */
- ptr++;
- } while(ptr < end);
- hc4->nextToUpdate = end;
+ matchIndex -= chainTable[matchIndex & 0xFFFF];
}
-#endif
return (int)ml;
}
-FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* startLimit, const BYTE* matchlimit, int longest, const BYTE** matchpos, const BYTE** startpos, const int maxNbAttempts)
+FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
+ LZ4HC_Data_Structure* hc4,
+ const BYTE* ip,
+ const BYTE* startLimit,
+ const BYTE* matchlimit,
+ int longest,
+ const BYTE** matchpos,
+ const BYTE** startpos,
+ const int maxNbAttempts)
{
- U16* const chainTable = hc4->chainTable;
- HTYPE* const HashTable = hc4->hashTable;
- INITBASE(base,hc4->base);
- const BYTE* ref;
+ U16* const chainTable = hc4->chainTable;
+ U32* const HashTable = hc4->hashTable;
+ const BYTE* const base = hc4->base;
+ const BYTE* match;
+ int matchIndex;
+ const int idxLow = (ip-base) > 64 KB ? (ip-base) - 64 KB : 0;
int nbAttempts = maxNbAttempts;
int delta = (int)(ip-startLimit);
/* First Match */
LZ4HC_Insert(hc4, ip);
- ref = HASH_POINTER(ip);
+ matchIndex = HashTable[HASH_VALUE(ip)];
- while (((U32)(ip-ref) <= MAX_DISTANCE) && (nbAttempts))
+ while ((matchIndex>idxLow) && (nbAttempts))
{
nbAttempts--;
- if (*(startLimit + longest) == *(ref - delta + longest))
- if (A32(ref) == A32(ip))
+ match = base + matchIndex;
+ if (*(startLimit + longest) == *(match - delta + longest))
+ if (A32(match) == A32(ip))
{
-#if 1
- const BYTE* reft = ref+MINMATCH;
- const BYTE* ipt = ip+MINMATCH;
- const BYTE* startt = ip;
-
- while (ipt<matchlimit-(STEPSIZE-1))
- {
- size_t diff = AARCH(reft) ^ AARCH(ipt);
- if (!diff) { ipt+=STEPSIZE; reft+=STEPSIZE; continue; }
- ipt += LZ4_NbCommonBytes(diff);
- goto _endCount;
- }
- if (LZ4_ARCH64) if ((ipt<(matchlimit-3)) && (A32(reft) == A32(ipt))) { ipt+=4; reft+=4; }
- if ((ipt<(matchlimit-1)) && (A16(reft) == A16(ipt))) { ipt+=2; reft+=2; }
- if ((ipt<matchlimit) && (*reft == *ipt)) ipt++;
-_endCount:
- reft = ref;
-#else
- /* Easier for code maintenance, but unfortunately slower too */
const BYTE* startt = ip;
- const BYTE* reft = ref;
- const BYTE* ipt = ip + MINMATCH + LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit);
-#endif
+ const BYTE* tmpMatch = match;
+ const BYTE* ipt = ip + MINMATCH + LZ4HC_CommonLength(ip+MINMATCH, match+MINMATCH, matchlimit);
- while ((startt>startLimit) && (reft > hc4->inputBuffer) && (startt[-1] == reft[-1])) {startt--; reft--;}
+ while ((startt>startLimit) && (tmpMatch > hc4->inputBuffer) && (startt[-1] == tmpMatch[-1])) {startt--; tmpMatch--;}
if ((ipt-startt) > longest)
{
longest = (int)(ipt-startt);
- *matchpos = reft;
+ *matchpos = tmpMatch;
*startpos = startt;
}
}
- ref = GETNEXT(ref);
+ matchIndex -= chainTable[matchIndex & 0xFFFF];
}
return longest;
@@ -574,7 +481,7 @@ FORCE_INLINE int LZ4HC_encodeSequence (
BYTE** op,
const BYTE** anchor,
int matchLength,
- const BYTE* ref,
+ const BYTE* match,
limitedOutput_directive limitedOutputBuffer,
BYTE* oend)
{
@@ -592,7 +499,7 @@ FORCE_INLINE int LZ4HC_encodeSequence (
LZ4_BLINDCOPY(*anchor, *op, length);
/* Encode Offset */
- LZ4_WRITE_LITTLEENDIAN_16(*op,(U16)(*ip-ref));
+ LZ4_WRITE_LITTLEENDIAN_16(*op,(U16)(*ip-match));
/* Encode MatchLength */
length = (int)(matchLength-MINMATCH);
@@ -629,7 +536,7 @@ static int LZ4HC_compress_generic (
BYTE* op = (BYTE*) dest;
BYTE* const oend = op + maxOutputSize;
- const int maxNbAttempts = compressionLevel > MAX_COMPRESSION_LEVEL ? 1 << MAX_COMPRESSION_LEVEL : compressionLevel ? 1<<(compressionLevel-1) : 1<<LZ4HC_DEFAULT_COMPRESSIONLEVEL;
+ unsigned maxNbAttempts;
int ml, ml2, ml3, ml0;
const BYTE* ref=NULL;
const BYTE* start2=NULL;
@@ -640,6 +547,10 @@ static int LZ4HC_compress_generic (
const BYTE* ref0;
+ /* init */
+ if (compressionLevel > MAX_COMPRESSION_LEVEL) compressionLevel = MAX_COMPRESSION_LEVEL;
+ if (compressionLevel == 0) compressionLevel = LZ4HC_DEFAULT_COMPRESSIONLEVEL;
+ maxNbAttempts = 1 << compressionLevel;
/* Ensure blocks follow each other */
if (ip != ctx->end) return 0;
ctx->end += inputSize;
@@ -794,7 +705,6 @@ _Search3:
ml2 = ml3;
goto _Search3;
-
}
/* Encode Last Literals */
@@ -814,28 +724,18 @@ _Search3:
int LZ4_compressHC2(const char* source, char* dest, int inputSize, int compressionLevel)
{
- void* ctx = LZ4_createHC(source);
- int result;
- if (ctx==NULL) return 0;
-
- result = LZ4HC_compress_generic (ctx, source, dest, inputSize, 0, compressionLevel, noLimit);
-
- LZ4_freeHC(ctx);
- return result;
+ LZ4HC_Data_Structure ctx;
+ LZ4HC_init(&ctx, (const BYTE*)source);
+ return LZ4HC_compress_generic (&ctx, source, dest, inputSize, 0, compressionLevel, noLimit);
}
int LZ4_compressHC(const char* source, char* dest, int inputSize) { return LZ4_compressHC2(source, dest, inputSize, 0); }
int LZ4_compressHC2_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
{
- void* ctx = LZ4_createHC(source);
- int result;
- if (ctx==NULL) return 0;
-
- result = LZ4HC_compress_generic (ctx, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
-
- LZ4_freeHC(ctx);
- return result;
+ LZ4HC_Data_Structure ctx;
+ LZ4HC_init(&ctx, (const BYTE*)source);
+ return LZ4HC_compress_generic (&ctx, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
}
int LZ4_compressHC_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize)
@@ -853,7 +753,7 @@ int LZ4_sizeofStateHC() { return sizeof(LZ4HC_Data_Structure); }
int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel)
{
if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */
- LZ4_initHC ((LZ4HC_Data_Structure*)state, (const BYTE*)source);
+ LZ4HC_init ((LZ4HC_Data_Structure*)state, (const BYTE*)source);
return LZ4HC_compress_generic (state, source, dest, inputSize, 0, compressionLevel, noLimit);
}
@@ -864,7 +764,7 @@ int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int
int LZ4_compressHC2_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
{
if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */
- LZ4_initHC ((LZ4HC_Data_Structure*)state, (const BYTE*)source);
+ LZ4HC_init ((LZ4HC_Data_Structure*)state, (const BYTE*)source);
return LZ4HC_compress_generic (state, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
}
@@ -872,26 +772,124 @@ int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, c
{ return LZ4_compressHC2_limitedOutput_withStateHC (state, source, dest, inputSize, maxOutputSize, 0); }
-/****************************
- Stream functions
-****************************/
+/**************************************
+ Experimental Streaming Functions
+**************************************/
+/* allocation */
+LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); }
+int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { free(LZ4_streamHCPtr); return 0; };
-int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize)
+
+/* initialization */
+void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
{
- return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, 0, noLimit);
+ LZ4_STATIC_ASSERT(sizeof(LZ4HC_Data_Structure) <= LZ4_STREAMHCSIZE); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
+ ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base = NULL;
+ ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel = (unsigned)compressionLevel;
}
-int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel)
+int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize)
{
- return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, compressionLevel, noLimit);
+ LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) dictionary);
+ LZ4HC_Insert ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*)dictionary +(dictSize-3));
+ ((LZ4HC_Data_Structure*) LZ4_streamHCPtr)->end = (const BYTE*)dictionary + dictSize;
+ return 1;
+}
+
+
+/* compression */
+
+int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize)
+{
+ if (((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base == NULL) LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) source);
+ return LZ4HC_compress_generic (LZ4_streamHCPtr, source, dest, inputSize, 0, ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel, noLimit);
+}
+
+int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize)
+{
+ if (((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base == NULL) LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) source);
+ return LZ4HC_compress_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel, limitedOutput);
+}
+
+
+/* dictionary saving */
+
+int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize)
+{
+ LZ4HC_Data_Structure* sp = (LZ4HC_Data_Structure*)LZ4_streamHCPtr;
+ if (dictSize > 64 KB) dictSize = 64 KB;
+ if (dictSize < 0) dictSize = 0;
+ if (dictSize > (sp->end - sp->base)) dictSize = sp->end - sp->base;
+ memcpy(safeBuffer, sp->end - dictSize, dictSize);
+ LZ4_loadDictHC(LZ4_streamHCPtr, safeBuffer, dictSize);
+ return dictSize;
}
+
+
+/***********************************
+ Deprecated Streaming functions
+***********************************/
+int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; }
+
+int LZ4_resetStreamStateHC(void* state, const char* inputBuffer)
+{
+ if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */
+ LZ4HC_init((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer);
+ return 0;
+}
+
+void* LZ4_createHC (const char* inputBuffer)
+{
+ void* hc4 = ALLOCATOR(sizeof(LZ4HC_Data_Structure));
+ LZ4HC_init ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer);
+ return hc4;
+}
+
+int LZ4_freeHC (void* LZ4HC_Data)
+{
+ FREEMEM(LZ4HC_Data);
+ return (0);
+}
+
+/*
+int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize)
+{
+ return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, 0, noLimit);
+}
int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize)
{
return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, 0, limitedOutput);
}
+*/
+
+int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel)
+{
+ return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, compressionLevel, noLimit);
+}
int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
{
return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
}
+
+char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
+{
+ LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data;
+ size_t distance = (hc4->end - 64 KB) - hc4->inputBuffer;
+
+ if (hc4->end <= hc4->inputBuffer + 64 KB) return (char*)(hc4->end); /* no update : less than 64KB within buffer */
+
+ distance = (distance >> 16) << 16; /* Must be a multiple of 64 KB */
+ LZ4HC_Insert(hc4, hc4->end - MINMATCH);
+ memcpy((void*)(hc4->end - 64 KB - distance), (const void*)(hc4->end - 64 KB), 64 KB);
+ hc4->base -= distance;
+ if ((U32)(hc4->inputBuffer - hc4->base) > 1 GB + 64 KB) /* Avoid overflow */
+ {
+ int i;
+ hc4->base += 1 GB;
+ for (i=0; i<HASHTABLESIZE; i++) hc4->hashTable[i] -= 1 GB;
+ }
+ hc4->end -= distance;
+ return (char*)(hc4->end);
+}
diff --git a/lz4hc.h b/lz4hc.h
index deb2394..29a05f5 100644
--- a/lz4hc.h
+++ b/lz4hc.h
@@ -74,7 +74,7 @@ int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize
*/
/* Note :
-Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license)
+ Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license)
*/
@@ -101,13 +101,63 @@ They just use the externally allocated memory area instead of allocating their o
*/
+
+
+/**************************************
+ Experimental Streaming Functions
+**************************************/
+#define LZ4_STREAMHCSIZE_U32 65546
+#define LZ4_STREAMHCSIZE (LZ4_STREAMHCSIZE_U32 * sizeof(unsigned int))
+typedef struct { unsigned int table[LZ4_STREAMHCSIZE_U32]; } LZ4_streamHC_t;
+
+/*
+This structure allows static allocation of LZ4 HC streaming state.
+State must then be initialized using LZ4_resetStreamHC() before first use.
+
+If you prefer dynamic allocation, please refer to functions below.
+*/
+
+LZ4_streamHC_t* LZ4_createStreamHC(void);
+int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr);
+
+/*
+These functions create and release memory for LZ4 HC streaming state.
+Newly created states are already initialized.
+Existing state space can be re-used anytime using LZ4_resetStreamHC().
+*/
+
+void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
+int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize);
+
+int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
+int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
+
+int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int maxDictSize);
+
+/*
+These functions compress data in successive blocks of any size, using previous blocks as dictionary.
+
+Before starting compression, state must be properly initialized, using LZ4_resetStreamHC().
+A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional).
+
+Then, use LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue() to compress each successive block.
+They work like usual LZ4_compressHC() or LZ4_compressHC_limitedOutput(), but use previous memory blocks to improve compression.
+Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression.
+
+If, for any reason, previous data block can't be preserved in memory for next block compression,
+you can still preserve it by moving it to a safer place,
+using LZ4_saveDictHC().
+*/
+
+
+
/**************************************
- Streaming Functions
+ Deprecated Streaming Functions
**************************************/
/* Note : these streaming functions still follows the older model */
void* LZ4_createHC (const char* inputBuffer);
-int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize);
-int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
+//int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize);
+//int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
int LZ4_freeHC (void* LZ4HC_Data);
diff --git a/programs/fullbench.c b/programs/fullbench.c
index f34c68c..4caea16 100644
--- a/programs/fullbench.c
+++ b/programs/fullbench.c
@@ -313,12 +313,12 @@ static int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inS
static int local_LZ4_compressHC_continue(const char* in, char* out, int inSize)
{
- return LZ4_compressHC_continue(ctx, in, out, inSize);
+ return LZ4_compressHC_continue((LZ4_streamHC_t*)ctx, in, out, inSize);
}
static int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize)
{
- return LZ4_compressHC_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize));
+ return LZ4_compressHC_limitedOutput_continue((LZ4_streamHC_t*)ctx, in, out, inSize, LZ4_compressBound(inSize));
}
static int local_LZ4F_compressFrame(const char* in, char* out, int inSize)
diff --git a/programs/fuzzer.c b/programs/fuzzer.c
index 283352c..d736b63 100644
--- a/programs/fuzzer.c
+++ b/programs/fuzzer.c
@@ -621,14 +621,102 @@ _output_error:
}
}
+static const unsigned testInputSize = 128 KB;
+static const unsigned testCompressedSize = 64 KB;
static void FUZ_unitTests(void)
{
+ const unsigned testNb = 0;
+ const unsigned seed = 0;
+ const unsigned cycleNb= 0;
+ char testInput[testInputSize];
+ char testCompressed[testCompressedSize];
+ char testVerify[testCompressedSize];
+ U32 randState = 0;
+
+ // Init
+ FUZ_fillCompressibleNoiseBuffer(testInput, testInputSize, 0.50, &randState);
+
+ // 32-bits address space overflow test
FUZ_AddressOverflow();
+
+ // LZ4 HC streaming tests
+ {
+ LZ4_streamHC_t* sp;
+ LZ4_streamHC_t sHC;
+ //XXH64_state_t xxh;
+ U64 crcOrig;
+ U64 crcNew;
+ int result;
+
+ // Allocation test
+ sp = LZ4_createStreamHC();
+ FUZ_CHECKTEST(sp==NULL, "LZ4_createStreamHC() allocation failed");
+ LZ4_freeStreamHC(sp);
+
+ // simple compression test
+ crcOrig = XXH64(testInput, testCompressedSize, 0);
+ LZ4_resetStreamHC(&sHC, 0);
+ result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1);
+ FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
+
+ result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize);
+ FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed");
+ crcNew = XXH64(testVerify, testCompressedSize, 0);
+ FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
+
+ // simple dictionary compression test
+ crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0);
+ LZ4_resetStreamHC(&sHC, 0);
+ LZ4_loadDictHC(&sHC, testInput, 64 KB);
+ result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1);
+ FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result);
+
+ result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 64 KB);
+ FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() dictionary decompression failed");
+ crcNew = XXH64(testVerify, testCompressedSize, 0);
+ FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption");
+
+ // dictionary multi compression test
+ {
+ int result1, result2;
+ int segSize = testCompressedSize / 2;
+ crcOrig = XXH64(testInput + segSize, 64 KB, 0);
+ LZ4_resetStreamHC(&sHC, 0);
+ LZ4_loadDictHC(&sHC, testInput, segSize);
+ result1 = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1);
+ FUZ_CHECKTEST(result1==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result);
+ result2 = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + 2*segSize, testCompressed+result1, segSize, segSize -1);
+ FUZ_CHECKTEST(result2==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result);
+
+ result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result1, segSize, testInput, segSize);
+ FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 1 failed");
+ result = LZ4_decompress_safe_usingDict(testCompressed+result1, testVerify+segSize, result2, segSize, testInput, 2*segSize);
+ FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 2 failed");
+ crcNew = XXH64(testVerify, testCompressedSize, 0);
+ FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption");
+ }
+
+ // remote dictionary compression test
+ crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0);
+ LZ4_resetStreamHC(&sHC, 0);
+ LZ4_loadDictHC(&sHC, testInput, 32 KB);
+ result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1);
+ //FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result);
+
+ result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 32 KB);
+ //FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() dictionary decompression failed");
+ crcNew = XXH64(testVerify, testCompressedSize, 0);
+ //FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption");
+ }
+
+ return;
+_output_error:
+ exit(1);
}
-int FUZ_usage(void)
+static int FUZ_usage(void)
{
DISPLAY( "Usage :\n");
DISPLAY( " %s [args]\n", programName);