summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rwxr-xr-xexamples/HCStreaming_ringBuffer.c90
-rw-r--r--examples/Makefile2
-rwxr-xr-xlz4.c145
-rw-r--r--lz4.h30
-rw-r--r--[-rwxr-xr-x]lz4hc.c13
-rw-r--r--[-rwxr-xr-x]lz4hc.h5
-rwxr-xr-x[-rw-r--r--]programs/Makefile2
-rwxr-xr-xprograms/fuzzer.c411
8 files changed, 435 insertions, 263 deletions
diff --git a/examples/HCStreaming_ringBuffer.c b/examples/HCStreaming_ringBuffer.c
index 3532fa3..53e14ca 100755
--- a/examples/HCStreaming_ringBuffer.c
+++ b/examples/HCStreaming_ringBuffer.c
@@ -3,7 +3,7 @@
/**************************************
- Compiler Options
+Compiler Options
**************************************/
#ifdef _MSC_VER /* Visual Studio */
# define _CRT_SECURE_NO_WARNINGS // for MSVC
@@ -18,7 +18,7 @@
/**************************************
- Includes
+Includes
**************************************/
#include "lz4hc.h"
#include "lz4.h"
@@ -30,8 +30,8 @@
enum {
MESSAGE_MAX_BYTES = 1024,
- RING_BUFFER_BYTES = 1024 * 256 + MESSAGE_MAX_BYTES,
- DICT_BYTES = 65536,
+ RING_BUFFER_BYTES = 1024 * 8 + MESSAGE_MAX_BYTES,
+ DEC_BUFFER_BYTES = RING_BUFFER_BYTES + MESSAGE_MAX_BYTES
};
@@ -59,9 +59,10 @@ void test_compress(FILE* outFp, FILE* inpFp)
static char inpBuf[RING_BUFFER_BYTES];
int inpOffset = 0;
+ unsigned done = 0;
for(;;)
- {
+ {
// Read random length ([1,MESSAGE_MAX_BYTES]) data to the ring buffer.
char* const inpPtr = &inpBuf[inpOffset];
const int randomLength = (rand() % MESSAGE_MAX_BYTES) + 1;
@@ -71,11 +72,13 @@ void test_compress(FILE* outFp, FILE* inpFp)
{
char cmpBuf[LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)];
const int cmpBytes = LZ4_compressHC_continue(lz4Stream, inpPtr, cmpBuf, inpBytes);
+
if(cmpBytes <= 0) break;
write_int32(outFp, cmpBytes);
write_bin(outFp, cmpBuf, cmpBytes);
inpOffset += inpBytes;
+ done += inpBytes;
// Wraparound the ringbuffer offset
if(inpOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES)
@@ -89,22 +92,26 @@ void test_compress(FILE* outFp, FILE* inpFp)
void test_decompress(FILE* outFp, FILE* inpFp)
{
- static char decBuf[RING_BUFFER_BYTES];
+ static char decBuf[DEC_BUFFER_BYTES];
int decOffset = 0;
LZ4_streamDecode_t lz4StreamDecode_body = { 0 };
LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;
+ unsigned done = 0;
- for(;;) {
- int cmpBytes = 0;
+ for(;;)
+ {
+ int cmpBytes = 0;
char cmpBuf[LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)];
{
const size_t r0 = read_int32(inpFp, &cmpBytes);
- size_t r1;
- if(r0 != 1 || cmpBytes <= 0) break;
+ size_t r1;
+ if(r0 != 1 || cmpBytes <= 0)
+ break;
r1 = read_bin(inpFp, cmpBuf, cmpBytes);
- if(r1 != (size_t) cmpBytes) break;
+ if(r1 != (size_t) cmpBytes)
+ break;
}
{
@@ -112,38 +119,56 @@ void test_decompress(FILE* outFp, FILE* inpFp)
const int decBytes = LZ4_decompress_safe_continue(
lz4StreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES);
if(decBytes <= 0)
- break;
+ break;
+
+ done += decBytes;
decOffset += decBytes;
write_bin(outFp, decPtr, decBytes);
// Wraparound the ringbuffer offset
- if(decOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES) decOffset = 0;
+ if(decOffset >= DEC_BUFFER_BYTES - MESSAGE_MAX_BYTES)
+ decOffset = 0;
}
}
}
-int compare(FILE* f0, FILE* f1)
+// Compare 2 files content
+// return 0 if identical
+// return ByteNb>0 if different
+size_t compare(FILE* f0, FILE* f1)
{
- int result = 0;
+ size_t result = 1;
- while(0 == result) {
+ for (;;)
+ {
char b0[65536];
char b1[65536];
const size_t r0 = fread(b0, 1, sizeof(b0), f0);
const size_t r1 = fread(b1, 1, sizeof(b1), f1);
- result = (int) r0 - (int) r1;
+ if ((r0==0) && (r1==0)) return 0; // success
- if(0 == r0 || 0 == r1) {
- break;
+ if (r0 != r1)
+ {
+ size_t smallest = r0;
+ if (r1<r0) smallest = r1;
+ result += smallest;
+ return result;
}
- if(0 == result) {
- result = memcmp(b0, b1, r0);
+
+ if (memcmp(b0, b1, r0))
+ {
+ unsigned errorPos = 0;
+ while ((errorPos < r0) && (b0[errorPos]==b1[errorPos])) errorPos++;
+ result += errorPos;
+ return result;
}
+
+ result += sizeof(b0);
}
- return result;
+ return 1;
}
@@ -152,15 +177,20 @@ int main(int argc, char** argv)
char inpFilename[256] = { 0 };
char lz4Filename[256] = { 0 };
char decFilename[256] = { 0 };
+ unsigned fileID = 1;
+ unsigned pause = 0;
+
if(argc < 2) {
printf("Please specify input filename\n");
return 0;
}
- snprintf(inpFilename, 256, "%s", argv[1]);
- snprintf(lz4Filename, 256, "%s.lz4s-%d", argv[1], 0);
- snprintf(decFilename, 256, "%s.lz4s-%d.dec", argv[1], 0);
+ if (!strcmp(argv[1], "-p")) pause = 1, fileID = 2;
+
+ snprintf(inpFilename, 256, "%s", argv[fileID]);
+ snprintf(lz4Filename, 256, "%s.lz4s-%d", argv[fileID], 9);
+ snprintf(decFilename, 256, "%s.lz4s-%d.dec", argv[fileID], 9);
printf("inp = [%s]\n", inpFilename);
printf("lz4 = [%s]\n", lz4Filename);
@@ -193,16 +223,22 @@ int main(int argc, char** argv)
FILE* inpFp = fopen(inpFilename, "rb");
FILE* decFp = fopen(decFilename, "rb");
- const int cmp = compare(inpFp, decFp);
+ const size_t cmp = compare(inpFp, decFp);
if(0 == cmp) {
printf("Verify : OK\n");
} else {
- printf("Verify : NG\n");
+ printf("Verify : NG : error at pos %u\n", (unsigned)cmp-1);
}
fclose(decFp);
fclose(inpFp);
}
+ if (pause)
+ {
+ printf("Press enter to continue ...\n");
+ getchar();
+ }
+
return 0;
}
diff --git a/examples/Makefile b/examples/Makefile
index b5f9a7e..4474f59 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -83,7 +83,7 @@ test : all
./lineCompress$(EXT) $(TESTFILE)
clean:
- @rm -f core *.o *.dec *-0 *-8192 *.lz4s \
+ @rm -f core *.o *.dec *-0 *-9 *-8192 *.lz4s \
printVersion$(EXT) doubleBuffer$(EXT) ringBuffer$(EXT) ringBufferHC$(EXT) lineCompress$(EXT)
@echo Cleaning completed
diff --git a/lz4.c b/lz4.c
index 8adf96a..e7f45d6 100755
--- a/lz4.c
+++ b/lz4.c
@@ -885,20 +885,20 @@ FORCE_INLINE int LZ4_decompress_generic(
int partialDecoding, /* full, partial */
int targetOutputSize, /* only used if partialDecoding==partial */
int dict, /* noDict, withPrefix64k, usingExtDict */
- const char* const dictStart, /* only if dict==usingExtDict */
- int dictSize /* note : = 0 if noDict */
+ const BYTE* const lowPrefix, /* == dest if dict == noDict */
+ const BYTE* const dictStart, /* only if dict==usingExtDict */
+ const size_t dictSize /* note : = 0 if noDict */
)
{
/* Local Variables */
const BYTE* restrict ip = (const BYTE*) source;
- const BYTE* ref;
const BYTE* const iend = ip + inputSize;
BYTE* op = (BYTE*) dest;
BYTE* const oend = op + outputSize;
BYTE* cpy;
BYTE* oexit = op + targetOutputSize;
- const BYTE* const lowLimit = (const BYTE*)dest - dictSize;
+ const BYTE* const lowLimit = lowPrefix - dictSize;
const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize;
const size_t dec32table[] = {4, 1, 2, 1, 4, 4, 4, 4};
@@ -919,8 +919,9 @@ FORCE_INLINE int LZ4_decompress_generic(
{
unsigned token;
size_t length;
+ const BYTE* match;
- /* get runlength */
+ /* get literal length */
token = *ip++;
if ((length=(token>>ML_BITS)) == RUN_MASK)
{
@@ -931,8 +932,8 @@ FORCE_INLINE int LZ4_decompress_generic(
length += s;
}
while (likely((endOnInput)?ip<iend-RUN_MASK:1) && (s==255));
- if ((safeDecode) && LZ4_32BITS && unlikely((size_t)(op+length)<(size_t)(op))) goto _output_error; /* overflow detection */
- if ((safeDecode) && LZ4_32BITS && unlikely((size_t)(ip+length)<(size_t)(ip))) goto _output_error; /* overflow detection */
+ if ((safeDecode) && unlikely((size_t)(op+length)<(size_t)(op))) goto _output_error; /* overflow detection */
+ if ((safeDecode) && unlikely((size_t)(ip+length)<(size_t)(ip))) goto _output_error; /* overflow detection */
}
/* copy literals */
@@ -958,8 +959,8 @@ FORCE_INLINE int LZ4_decompress_generic(
LZ4_WILDCOPY(op, ip, cpy); ip -= (op-cpy); op = cpy;
/* get offset */
- LZ4_READ_LITTLEENDIAN_16(ref,cpy,ip); ip+=2;
- if ((checkOffset) && (unlikely(ref < lowLimit))) goto _output_error; /* Error : offset outside destination buffer */
+ LZ4_READ_LITTLEENDIAN_16(match,cpy,ip); ip+=2;
+ if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside destination buffer */
/* get matchlength */
if ((length=(token&ML_MASK)) == ML_MASK)
@@ -971,36 +972,38 @@ FORCE_INLINE int LZ4_decompress_generic(
s = *ip++;
length += s;
} while (s==255);
- if ((safeDecode) && LZ4_32BITS && unlikely((size_t)(op+length)<(size_t)op)) goto _output_error; /* overflow detection */
+ if ((safeDecode) && unlikely((size_t)(op+length)<(size_t)op)) goto _output_error; /* overflow detection */
}
length += MINMATCH;
/* check external dictionary */
- if ((dict==usingExtDict) && (ref < (BYTE* const)dest))
+ if ((dict==usingExtDict) && (match < lowPrefix))
{
- if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error;
+ if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */
- if (length <= (size_t)(dest-(char*)ref))
+ if (length <= (size_t)(lowPrefix-match))
{
- ref = dictEnd - (dest-(char*)ref);
- memcpy(op, ref, length);
+ /* match can be copied as a single segment from external dictionary */
+ match = dictEnd - (lowPrefix-match);
+ memcpy(op, match, length);
op += length;
}
else
{
- size_t copySize = (size_t)(dest-(char*)ref);
+ /* match encompass external dictionary and current segment */
+ size_t copySize = (size_t)(lowPrefix-match);
memcpy(op, dictEnd - copySize, copySize);
op += copySize;
copySize = length - copySize;
- if (copySize > (size_t)((char*)op-dest)) /* overlap */
+ if (copySize > (size_t)(op-lowPrefix)) /* overlap within current segment */
{
BYTE* const endOfMatch = op + copySize;
- const BYTE* copyFrom = (BYTE*)dest;
+ const BYTE* copyFrom = lowPrefix;
while (op < endOfMatch) *op++ = *copyFrom++;
}
else
{
- memcpy(op, dest, copySize);
+ memcpy(op, lowPrefix, copySize);
op += copySize;
}
}
@@ -1009,25 +1012,25 @@ FORCE_INLINE int LZ4_decompress_generic(
/* copy repeated sequence */
cpy = op + length;
- if (unlikely((op-ref)<(int)STEPSIZE))
+ if (unlikely((op-match)<(int)STEPSIZE))
{
- const size_t dec64 = dec64table[op-ref];
- op[0] = ref[0];
- op[1] = ref[1];
- op[2] = ref[2];
- op[3] = ref[3];
- ref += dec32table[op-ref];
- A32(op+4) = A32(ref);
- op += 8; ref -= dec64;
- } else { LZ4_COPY8(op,ref); }
+ const size_t dec64 = dec64table[op-match];
+ op[0] = match[0];
+ op[1] = match[1];
+ op[2] = match[2];
+ op[3] = match[3];
+ match += dec32table[op-match];
+ A32(op+4) = A32(match);
+ op += 8; match -= dec64;
+ } else { LZ4_COPY8(op,match); }
if (unlikely(cpy>oend-12))
{
if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last 5 bytes must be literals */
- if (op<oend-COPYLENGTH) LZ4_WILDCOPY(op, ref, (oend-COPYLENGTH));
- while(op<cpy) *op++=*ref++;
+ if (op<oend-COPYLENGTH) LZ4_WILDCOPY(op, match, (oend-COPYLENGTH));
+ while(op<cpy) *op++=*match++;
}
- else LZ4_WILDCOPY(op, ref, cpy);
+ else LZ4_WILDCOPY(op, match, cpy);
op=cpy; /* correction */
}
@@ -1045,25 +1048,28 @@ _output_error:
int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize)
{
- return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, NULL, 0);
+ return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, (BYTE*)dest, NULL, 0);
}
int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize)
{
- return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, NULL, 0);
+ return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0);
}
int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
{
- return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, NULL, 64 KB);
+ return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB);
}
+
/* streaming decompression functions */
typedef struct
{
- const char* dictionary;
- int dictSize;
+ BYTE* externalDict;
+ size_t extDictSize;
+ BYTE* prefixEnd;
+ size_t prefixSize;
} LZ4_streamDecode_t_internal;
/*
@@ -1074,7 +1080,6 @@ typedef struct
LZ4_streamDecode_t* LZ4_createStreamDecode(void)
{
LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(sizeof(U32), LZ4_STREAMDECODESIZE_U32);
- MEM_INIT(lz4s, 0, LZ4_STREAMDECODESIZE);
return lz4s;
}
@@ -1094,8 +1099,10 @@ int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream)
int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize)
{
LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode;
- lz4sd->dictionary = dictionary;
- lz4sd->dictSize = dictSize;
+ lz4sd->prefixSize = (size_t) dictSize;
+ lz4sd->prefixEnd = (BYTE*) dictionary + dictSize;
+ lz4sd->externalDict = NULL;
+ lz4sd->extDictSize = 0;
return 1;
}
@@ -1104,23 +1111,32 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti
These decoding functions allow decompression of multiple blocks in "streaming" mode.
Previously decoded blocks must still be available at the memory position where they were decoded.
If it's not possible, save the relevant part of decoded data into a safe buffer,
- and indicate where it stands using LZ4_setDictDecode()
+ and indicate where it stands using LZ4_setStreamDecode()
*/
int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize)
{
LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode;
int result;
- result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, lz4sd->dictionary, lz4sd->dictSize);
- if (result <= 0) return result;
- if (lz4sd->dictionary + lz4sd->dictSize == dest)
+ if (lz4sd->prefixEnd == (BYTE*)dest)
{
- lz4sd->dictSize += result;
+ result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
+ endOnInputSize, full, 0,
+ usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize);
+ if (result <= 0) return result;
+ lz4sd->prefixSize += result;
+ lz4sd->prefixEnd += result;
}
else
{
- lz4sd->dictionary = dest;
- lz4sd->dictSize = result;
+ lz4sd->extDictSize = lz4sd->prefixSize;
+ lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize;
+ result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
+ endOnInputSize, full, 0,
+ usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize);
+ if (result <= 0) return result;
+ lz4sd->prefixSize = result;
+ lz4sd->prefixEnd = (BYTE*)dest + result;
}
return result;
@@ -1131,16 +1147,25 @@ int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch
LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode;
int result;
- result = LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, usingExtDict, lz4sd->dictionary, lz4sd->dictSize);
- if (result <= 0) return result;
- if (lz4sd->dictionary + lz4sd->dictSize == dest)
+ if (lz4sd->prefixEnd == (BYTE*)dest)
{
- lz4sd->dictSize += result;
+ result = LZ4_decompress_generic(source, dest, 0, originalSize,
+ endOnOutputSize, full, 0,
+ usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize);
+ if (result <= 0) return result;
+ lz4sd->prefixSize += originalSize;
+ lz4sd->prefixEnd += originalSize;
}
else
{
- lz4sd->dictionary = dest;
- lz4sd->dictSize = result;
+ lz4sd->extDictSize = lz4sd->prefixSize;
+ lz4sd->externalDict = (BYTE*)dest - lz4sd->extDictSize;
+ result = LZ4_decompress_generic(source, dest, 0, originalSize,
+ endOnOutputSize, full, 0,
+ usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize);
+ if (result <= 0) return result;
+ lz4sd->prefixSize = originalSize;
+ lz4sd->prefixEnd = (BYTE*)dest + originalSize;
}
return result;
@@ -1157,28 +1182,26 @@ Advanced decoding functions :
FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize)
{
if (dictSize==0)
- return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, NULL, 64 KB);
+ return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 64 KB);
if ((dictStart+dictSize == dest) && (dictSize >= (int)(64 KB - 1)))
- return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, NULL, 64 KB);
- return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, dictStart, dictSize);
+ return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 64 KB);
+ return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (BYTE*)dictStart, dictSize);
}
int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
{
- //return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, dictStart, dictSize);
return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize);
}
int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize)
{
- //return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, usingExtDict, dictStart, dictSize);
return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize);
}
/* debug function */
int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
{
- return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, dictStart, dictSize);
+ return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (BYTE*)dictStart, dictSize);
}
@@ -1258,10 +1281,10 @@ int LZ4_compress_limitedOutput_withState (void* state, const char* source, char*
int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize)
{
- return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, NULL, 64 KB);
+ return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB);
}
int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize)
{
- return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, NULL, 64 KB);
+ return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB);
}
diff --git a/lz4.h b/lz4.h
index 7ad736f..eb96067 100644
--- a/lz4.h
+++ b/lz4.h
@@ -47,8 +47,8 @@ extern "C" {
Version
**************************************/
#define LZ4_VERSION_MAJOR 1 /* for major interface/format changes */
-#define LZ4_VERSION_MINOR 3 /* for minor interface/format changes */
-#define LZ4_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */
+#define LZ4_VERSION_MINOR 4 /* for minor interface/format changes */
+#define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */
#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
int LZ4_versionNumber (void);
@@ -234,7 +234,7 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_streamPtr, char* safeBuffer, int dictSize);
Experimental Streaming Decompression Functions
************************************************/
-#define LZ4_STREAMDECODESIZE_U32 4
+#define LZ4_STREAMDECODESIZE_U32 8
#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U32 * sizeof(unsigned int))
/*
* LZ4_streamDecode_t
@@ -244,29 +244,29 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_streamPtr, char* safeBuffer, int dictSize);
typedef struct { unsigned int table[LZ4_STREAMDECODESIZE_U32]; } LZ4_streamDecode_t;
/*
+ * If you prefer dynamic allocation methods,
+ * LZ4_createStreamDecode will allocate and initialize an LZ4_streamDecode_t structure
+ * LZ4_freeStreamDecode releases its memory.
+ */
+LZ4_streamDecode_t* LZ4_createStreamDecode(void);
+int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
+
+/*
* LZ4_setStreamDecode
* Use this function to instruct where to find the dictionary.
* This function can be used to specify a static dictionary,
* or to instruct where to find some previously decoded data saved into a different memory space.
- * Setting a size of 0 is allowed (same effect as no dictionary).
+ * Setting a size of 0 is allowed (same effect as no dictionary, same effect as reset).
* Return : 1 if OK, 0 if error
*/
int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
/*
- * If you prefer dynamic allocation methods,
- * LZ4_createStreamDecode will allocate and initialize an LZ4_streamDecode_t structure
- * LZ4_freeStreamDecode releases its memory.
- */
-LZ4_streamDecode_t* LZ4_createStreamDecode(void);
-int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
-
-/*
*_continue() :
These decoding functions allow decompression of multiple blocks in "streaming" mode.
- Previously decoded blocks must still be available at the memory position where they were decoded.
- If it's not possible, save the relevant part of decoded data into a safe buffer,
- and indicate where its new address using LZ4_setStreamDecode()
+ Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
+ If this condition is not possible, save the relevant part of decoded data into a safe buffer,
+ and indicate where is its new address using LZ4_setStreamDecode()
*/
int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize);
diff --git a/lz4hc.c b/lz4hc.c
index ce273de..8fc9606 100755..100644
--- a/lz4hc.c
+++ b/lz4hc.c
@@ -546,14 +546,14 @@ FORCE_INLINE int LZ4HC_encodeSequence (
BYTE** op,
const BYTE** anchor,
int matchLength,
- const BYTE* match,
+ const BYTE* const match,
limitedOutput_directive limitedOutputBuffer,
BYTE* oend)
{
int length;
BYTE* token;
- //printf("literal : %u -- match : %u \n", (U32)(*ip - *anchor), (U32)matchLength); // debug
+ //if (debug) printf("literal : %u -- match : %u -- offset : %u\n", (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match)); // debug
/* Encode Literal length */
length = (int)(*ip - *anchor);
@@ -880,8 +880,9 @@ static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* dsPtr,
const BYTE* sourceEnd = (const BYTE*) source + inputSize;
const BYTE* dictBegin = dsPtr->dictBase + dsPtr->lowLimit;
const BYTE* dictEnd = dsPtr->dictBase + dsPtr->dictLimit;
- if ((sourceEnd > dictBegin) && (sourceEnd < dictEnd))
+ if ((sourceEnd > dictBegin) && ((BYTE*)source < dictEnd))
{
+ if (sourceEnd > dictEnd) sourceEnd = dictEnd;
dsPtr->lowLimit = (U32)(sourceEnd - dsPtr->dictBase);
if (dsPtr->dictLimit - dsPtr->lowLimit < 4) dsPtr->lowLimit = dsPtr->dictLimit;
}
@@ -892,17 +893,11 @@ static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* dsPtr,
int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize)
{
- //if (((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base == NULL)
- // LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) source);
- //return LZ4HC_compress_generic (LZ4_streamHCPtr, source, dest, inputSize, 0, ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel, noLimit);
return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, 0, noLimit);
}
int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize)
{
- //if (((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base == NULL)
- // LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) source);
- //return LZ4HC_compress_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel, limitedOutput);
return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput);
}
diff --git a/lz4hc.h b/lz4hc.h
index 72979b5..5fbf398 100755..100644
--- a/lz4hc.h
+++ b/lz4hc.h
@@ -135,6 +135,7 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int maxDi
/*
These functions compress data in successive blocks of any size, using previous blocks as dictionary.
+One key assumption is that each previous block will remain read-accessible while compressing next block.
Before starting compression, state must be properly initialized, using LZ4_resetStreamHC().
A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional).
@@ -143,8 +144,8 @@ Then, use LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue() t
They work like usual LZ4_compressHC() or LZ4_compressHC_limitedOutput(), but use previous memory blocks to improve compression.
Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression.
-If, for any reason, previous data block can't be preserved in memory for next block compression,
-you can still preserve it by moving it to a safer place,
+If, for any reason, previous data block can't be preserved in memory during next compression block,
+you can save it to a safer memory space,
using LZ4_saveDictHC().
*/
diff --git a/programs/Makefile b/programs/Makefile
index 04694f2..e7e57a1 100644..100755
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -30,7 +30,7 @@
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
# ##########################################################################
-RELEASE=r123
+RELEASE=r124
DESTDIR?=
PREFIX ?= /usr
diff --git a/programs/fuzzer.c b/programs/fuzzer.c
index 225361b..07935fa 100755
--- a/programs/fuzzer.c
+++ b/programs/fuzzer.c
@@ -1,29 +1,29 @@
/*
- fuzzer.c - Fuzzer test tool for LZ4
- Copyright (C) Yann Collet 2012-2014
- GPL v2 License
-
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 2 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License along
- with this program; if not, write to the Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
- You can contact the author at :
- - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- - LZ4 source repository : http://code.google.com/p/lz4/
+fuzzer.c - Fuzzer test tool for LZ4
+Copyright (C) Yann Collet 2012-2014
+GPL v2 License
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+You can contact the author at :
+- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
+- LZ4 source repository : http://code.google.com/p/lz4/
*/
/**************************************
- Remove Visual warning messages
+Remove Visual warning messages
**************************************/
#define _CRT_SECURE_NO_WARNINGS // fgets
#ifdef _MSC_VER /* Visual Studio */
@@ -34,7 +34,7 @@
/**************************************
- Includes
+Includes
**************************************/
#include <stdlib.h>
#include <stdio.h> // fgets, sscanf
@@ -46,26 +46,26 @@
/**************************************
- Basic Types
+Basic Types
**************************************/
#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
# include <stdint.h>
- typedef uint8_t BYTE;
- typedef uint16_t U16;
- typedef uint32_t U32;
- typedef int32_t S32;
- typedef uint64_t U64;
+typedef uint8_t BYTE;
+typedef uint16_t U16;
+typedef uint32_t U32;
+typedef int32_t S32;
+typedef uint64_t U64;
#else
- typedef unsigned char BYTE;
- typedef unsigned short U16;
- typedef unsigned int U32;
- typedef signed int S32;
- typedef unsigned long long U64;
+typedef unsigned char BYTE;
+typedef unsigned short U16;
+typedef unsigned int U32;
+typedef signed int S32;
+typedef unsigned long long U64;
#endif
/**************************************
- Constants
+Constants
**************************************/
#ifndef LZ4_VERSION
# define LZ4_VERSION ""
@@ -87,7 +87,7 @@
/*****************************************
- Macros
+Macros
*****************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
@@ -97,7 +97,7 @@ static U32 g_time = 0;
/*********************************************************
- Fuzzer functions
+Fuzzer functions
*********************************************************/
static U32 FUZ_GetMilliStart(void)
{
@@ -178,88 +178,88 @@ static void FUZ_fillCompressibleNoiseBuffer(void* buffer, int bufferSize, double
#define BLOCKSIZE_I134 (32 MB)
static int FUZ_AddressOverflow(void)
{
- char* buffers[MAX_NB_BUFF_I134+1] = {0};
- int i, nbBuff=0;
- int highAddress = 0;
+ char* buffers[MAX_NB_BUFF_I134+1] = {0};
+ int i, nbBuff=0;
+ int highAddress = 0;
- printf("Overflow tests : ");
+ printf("Overflow tests : ");
- // Only possible in 32-bits
- if (sizeof(void*)==8)
- {
- printf("64 bits mode : no overflow \n");
- fflush(stdout);
- return 0;
- }
-
- buffers[0] = (char*)malloc(BLOCKSIZE_I134);
- buffers[1] = (char*)malloc(BLOCKSIZE_I134);
- if ((!buffers[0]) || (!buffers[1]))
- {
- printf("not enough memory for tests \n");
- return 0;
- }
- for (nbBuff=2; nbBuff < MAX_NB_BUFF_I134; nbBuff++)
- {
- printf("%3i \b\b\b\b", nbBuff);
- buffers[nbBuff] = (char*)malloc(BLOCKSIZE_I134);
- //printf("%08X ", (U32)(size_t)(buffers[nbBuff]));
- fflush(stdout);
-
- if (((size_t)buffers[nbBuff] > (size_t)0x80000000) && (!highAddress))
+ // Only possible in 32-bits
+ if (sizeof(void*)==8)
{
- printf("high address detected : ");
+ printf("64 bits mode : no overflow \n");
fflush(stdout);
- highAddress=1;
+ return 0;
}
- if (buffers[nbBuff]==NULL) goto _endOfTests;
+ buffers[0] = (char*)malloc(BLOCKSIZE_I134);
+ buffers[1] = (char*)malloc(BLOCKSIZE_I134);
+ if ((!buffers[0]) || (!buffers[1]))
{
- size_t sizeToGenerateOverflow = (size_t)(- ((size_t)buffers[nbBuff-1]) + 512);
- int nbOf255 = (int)((sizeToGenerateOverflow / 255) + 1);
- char* input = buffers[nbBuff-1];
- char* output = buffers[nbBuff];
- int r;
- input[0] = (char)0xF0; // Literal length overflow
- input[1] = (char)0xFF;
- input[2] = (char)0xFF;
- input[3] = (char)0xFF;
- for(i = 4; i <= nbOf255+4; i++) input[i] = (char)0xff;
- r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
- if (r>0) goto _overflowError;
- input[0] = (char)0x1F; // Match length overflow
- input[1] = (char)0x01;
- input[2] = (char)0x01;
- input[3] = (char)0x00;
- r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
- if (r>0) goto _overflowError;
-
- output = buffers[nbBuff-2]; // Reverse in/out pointer order
- input[0] = (char)0xF0; // Literal length overflow
- input[1] = (char)0xFF;
- input[2] = (char)0xFF;
- input[3] = (char)0xFF;
- r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
- if (r>0) goto _overflowError;
- input[0] = (char)0x1F; // Match length overflow
- input[1] = (char)0x01;
- input[2] = (char)0x01;
- input[3] = (char)0x00;
- r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
- if (r>0) goto _overflowError;
+ printf("not enough memory for tests \n");
+ return 0;
}
- }
+ for (nbBuff=2; nbBuff < MAX_NB_BUFF_I134; nbBuff++)
+ {
+ printf("%3i \b\b\b\b", nbBuff);
+ buffers[nbBuff] = (char*)malloc(BLOCKSIZE_I134);
+ //printf("%08X ", (U32)(size_t)(buffers[nbBuff]));
+ fflush(stdout);
+
+ if (((size_t)buffers[nbBuff] > (size_t)0x80000000) && (!highAddress))
+ {
+ printf("high address detected : ");
+ fflush(stdout);
+ highAddress=1;
+ }
+ if (buffers[nbBuff]==NULL) goto _endOfTests;
- nbBuff++;
+ {
+ size_t sizeToGenerateOverflow = (size_t)(- ((size_t)buffers[nbBuff-1]) + 512);
+ int nbOf255 = (int)((sizeToGenerateOverflow / 255) + 1);
+ char* input = buffers[nbBuff-1];
+ char* output = buffers[nbBuff];
+ int r;
+ input[0] = (char)0xF0; // Literal length overflow
+ input[1] = (char)0xFF;
+ input[2] = (char)0xFF;
+ input[3] = (char)0xFF;
+ for(i = 4; i <= nbOf255+4; i++) input[i] = (char)0xff;
+ r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
+ if (r>0) goto _overflowError;
+ input[0] = (char)0x1F; // Match length overflow
+ input[1] = (char)0x01;
+ input[2] = (char)0x01;
+ input[3] = (char)0x00;
+ r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
+ if (r>0) goto _overflowError;
+
+ output = buffers[nbBuff-2]; // Reverse in/out pointer order
+ input[0] = (char)0xF0; // Literal length overflow
+ input[1] = (char)0xFF;
+ input[2] = (char)0xFF;
+ input[3] = (char)0xFF;
+ r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
+ if (r>0) goto _overflowError;
+ input[0] = (char)0x1F; // Match length overflow
+ input[1] = (char)0x01;
+ input[2] = (char)0x01;
+ input[3] = (char)0x00;
+ r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
+ if (r>0) goto _overflowError;
+ }
+ }
+
+ nbBuff++;
_endOfTests:
- for (i=0 ; i<nbBuff; i++) free(buffers[i]);
- if (!highAddress) printf("high address not possible \n");
- else printf("all overflows correctly detected \n");
- return 0;
+ for (i=0 ; i<nbBuff; i++) free(buffers[i]);
+ if (!highAddress) printf("high address not possible \n");
+ else printf("all overflows correctly detected \n");
+ return 0;
_overflowError:
- printf("Address space overflow error !! \n");
- exit(1);
+ printf("Address space overflow error !! \n");
+ exit(1);
}
@@ -285,15 +285,15 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
char* decodedBuffer;
# define FUZ_max LZ4_COMPRESSBOUND(LEN)
int ret;
- unsigned cycleNb;
+ unsigned cycleNb;
# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \
- printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; }
+ printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; }
# define FUZ_DISPLAYTEST { testNb++; g_displayLevel<3 ? 0 : printf("%2u\b\b", testNb); if (g_displayLevel==4) fflush(stdout); }
void* stateLZ4 = malloc(LZ4_sizeofState());
void* stateLZ4HC = malloc(LZ4_sizeofStateHC());
void* LZ4continue;
LZ4_stream_t LZ4dict;
- LZ4_streamHC_t LZ4dictHC;
+ LZ4_streamHC_t LZ4dictHC;
U32 crcOrig, crcCheck;
U32 coreRandState = seed;
U32 randState = coreRandState ^ PRIME3;
@@ -315,9 +315,9 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
if (0) // some problems related to dictionary re-use; in this case, enable this loop
{
- int dictSize, blockSize, blockStart;
- char* dict;
- char* block;
+ int dictSize, blockSize, blockStart;
+ char* dict;
+ char* block;
FUZ_displayUpdate(cycleNb);
randState = coreRandState ^ PRIME3;
blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE;
@@ -494,19 +494,19 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
FUZ_CHECKTEST(ret, "LZ4_compress_limitedOutput should have failed (output buffer too small by 1 byte)");
FUZ_CHECKTEST(compressedBuffer[compressedSize-1], "LZ4_compress_limitedOutput overran output buffer")
- // Test HC compression with just one missing byte into output buffer => must fail
- FUZ_DISPLAYTEST;
+ // Test HC compression with just one missing byte into output buffer => must fail
+ FUZ_DISPLAYTEST;
compressedBuffer[HCcompressedSize-1] = 0;
ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize-1);
FUZ_CHECKTEST(ret, "LZ4_compressHC_limitedOutput should have failed (output buffer too small by 1 byte)");
FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-1], "LZ4_compressHC_limitedOutput overran output buffer")
- /* Dictionary tests */
+ /* Dictionary tests */
- // Compress using dictionary
- FUZ_DISPLAYTEST;
+ // Compress using dictionary
+ FUZ_DISPLAYTEST;
LZ4continue = LZ4_create (dict);
- LZ4_compress_continue ((LZ4_stream_t*)LZ4continue, dict, compressedBuffer, dictSize); // Just to fill hash tables
+ LZ4_compress_continue ((LZ4_stream_t*)LZ4continue, dict, compressedBuffer, dictSize); // Just to fill hash tables
blockContinueCompressedSize = LZ4_compress_continue ((LZ4_stream_t*)LZ4continue, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
free (LZ4continue);
@@ -557,7 +557,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size")
- crcCheck = XXH32(decodedBuffer, blockSize, 0);
+ crcCheck = XXH32(decodedBuffer, blockSize, 0);
if (crcCheck!=crcOrig)
{
int i=0;
@@ -571,7 +571,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size")
- crcCheck = XXH32(decodedBuffer, blockSize, 0);
+ crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
FUZ_DISPLAYTEST;
@@ -599,30 +599,30 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
}
// Compress HC using External dictionary
- FUZ_DISPLAYTEST;
- dict -= (FUZ_rand(&randState) & 7); // even bigger separation
- if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
- LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
- blockContinueCompressedSize = LZ4_compressHC_continue(&LZ4dictHC, block, compressedBuffer, blockSize);
- FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compressHC_continue failed");
+ FUZ_DISPLAYTEST;
+ dict -= (FUZ_rand(&randState) & 7); // even bigger separation
+ if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
+ LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
+ blockContinueCompressedSize = LZ4_compressHC_continue(&LZ4dictHC, block, compressedBuffer, blockSize);
+ FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compressHC_continue failed");
- FUZ_DISPLAYTEST;
- LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
- ret = LZ4_compressHC_limitedOutput_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
- FUZ_CHECKTEST(ret>0, "LZ4_compressHC_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer");
+ FUZ_DISPLAYTEST;
+ LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
+ ret = LZ4_compressHC_limitedOutput_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
+ FUZ_CHECKTEST(ret>0, "LZ4_compressHC_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer");
- FUZ_DISPLAYTEST;
- LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
- ret = LZ4_compressHC_limitedOutput_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
- FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
- FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer");
+ FUZ_DISPLAYTEST;
+ LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
+ ret = LZ4_compressHC_limitedOutput_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
+ FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
+ FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer");
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size")
- crcCheck = XXH32(decodedBuffer, blockSize, 0);
+ crcCheck = XXH32(decodedBuffer, blockSize, 0);
if (crcCheck!=crcOrig)
{
int i=0;
@@ -632,8 +632,8 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
- // ***** End of tests *** //
- // Fill stats
+ // ***** End of tests *** //
+ // Fill stats
bytes += blockSize;
cbytes += compressedSize;
hcbytes += HCcompressedSize;
@@ -648,7 +648,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
// unalloc
{
- int result = 0;
+ int result = 0;
_exit:
free(CNBuffer);
free(compressedBuffer);
@@ -663,8 +663,10 @@ _output_error:
}
}
+
#define testInputSize (128 KB)
#define testCompressedSize (64 KB)
+#define ringBufferSize (8 KB)
static void FUZ_unitTests(void)
{
@@ -674,7 +676,8 @@ static void FUZ_unitTests(void)
char testInput[testInputSize];
char testCompressed[testCompressedSize];
char testVerify[testInputSize];
- U32 randState = 0;
+ char ringBuffer[ringBufferSize];
+ U32 randState = 1;
// Init
FUZ_fillCompressibleNoiseBuffer(testInput, testInputSize, 0.50, &randState);
@@ -682,6 +685,75 @@ static void FUZ_unitTests(void)
// 32-bits address space overflow test
FUZ_AddressOverflow();
+ // LZ4 steraming tests
+ {
+ LZ4_stream_t* statePtr;
+ LZ4_stream_t streamingState;
+ U64 crcOrig;
+ U64 crcNew;
+ int result;
+
+ // Allocation test
+ statePtr = LZ4_createStream();
+ FUZ_CHECKTEST(statePtr==NULL, "LZ4_createStream() allocation failed");
+ LZ4_freeStream(statePtr);
+
+ // simple compression test
+ crcOrig = XXH64(testInput, testCompressedSize, 0);
+ LZ4_resetStream(&streamingState);
+ result = LZ4_compress_limitedOutput_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1);
+ FUZ_CHECKTEST(result==0, "LZ4_compress_limitedOutput_continue() compression failed");
+
+ result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize);
+ FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed");
+ crcNew = XXH64(testVerify, testCompressedSize, 0);
+ FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
+
+ // ring buffer test
+ {
+ XXH64_state_t xxhOrig;
+ XXH64_state_t xxhNew;
+ LZ4_streamDecode_t decodeState;
+ const U32 maxMessageSizeLog = 10;
+ const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1;
+ U32 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
+ U32 iNext = 0;
+ U32 rNext = 0;
+ U32 dNext = 0;
+ const U32 dBufferSize = ringBufferSize + maxMessageSizeMask;
+
+ XXH64_reset(&xxhOrig, 0);
+ XXH64_reset(&xxhNew, 0);
+ LZ4_resetStream(&streamingState);
+ LZ4_setStreamDecode(&decodeState, NULL, 0);
+
+ while (iNext + messageSize < testCompressedSize)
+ {
+ XXH64_update(&xxhOrig, testInput + iNext, messageSize);
+ crcOrig = XXH64_digest(&xxhOrig);
+
+ memcpy (ringBuffer + rNext, testInput + iNext, messageSize);
+ result = LZ4_compress_limitedOutput_continue(&streamingState, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
+ FUZ_CHECKTEST(result==0, "LZ4_compress_limitedOutput_continue() compression failed");
+
+ result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
+ FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed");
+
+ XXH64_update(&xxhNew, testVerify + dNext, messageSize);
+ crcNew = crcOrig = XXH64_digest(&xxhNew);
+ FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
+
+ // prepare next message
+ iNext += messageSize;
+ rNext += messageSize;
+ dNext += messageSize;
+ messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
+ if (rNext + messageSize > ringBufferSize) rNext = 0;
+ if (dNext + messageSize > dBufferSize) dNext = 0;
+ }
+ }
+ }
+
// LZ4 HC streaming tests
{
LZ4_streamHC_t* sp;
@@ -780,12 +852,12 @@ static void FUZ_unitTests(void)
FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %i failed", segNb);
XXH64_update(&crcNewState, dst, segSize);
crcNew = XXH64_digest(&crcNewState);
- if (crcOrig!=crcNew)
- {
- size_t c=0;
- while (dst[c] == testInput[segStart+c]) c++;
- DISPLAY("Bad decompression at %u / %u \n", (U32)c, (U32)segSize);
- }
+ if (crcOrig!=crcNew)
+ {
+ size_t c=0;
+ while (dst[c] == testInput[segStart+c]) c++;
+ DISPLAY("Bad decompression at %u / %u \n", (U32)c, (U32)segSize);
+ }
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() part %i corruption", segNb);
dict = dst;
@@ -799,6 +871,51 @@ static void FUZ_unitTests(void)
segSize = (FUZ_rand(&randState) & 8191);
}
}
+
+ // ring buffer test
+ {
+ XXH64_state_t xxhOrig;
+ XXH64_state_t xxhNew;
+ LZ4_streamDecode_t decodeState;
+ const U32 maxMessageSizeLog = 10;
+ const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1;
+ U32 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
+ U32 iNext = 0;
+ U32 rNext = 0;
+ U32 dNext = 0;
+ const U32 dBufferSize = ringBufferSize + maxMessageSizeMask;
+
+ XXH64_reset(&xxhOrig, 0);
+ XXH64_reset(&xxhNew, 0);
+ LZ4_resetStreamHC(&sHC, 0);
+ LZ4_setStreamDecode(&decodeState, NULL, 0);
+
+ while (iNext + messageSize < testCompressedSize)
+ {
+ XXH64_update(&xxhOrig, testInput + iNext, messageSize);
+ crcOrig = XXH64_digest(&xxhOrig);
+
+ memcpy (ringBuffer + rNext, testInput + iNext, messageSize);
+ result = LZ4_compressHC_limitedOutput_continue(&sHC, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
+ FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
+
+ result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
+ FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed");
+
+ XXH64_update(&xxhNew, testVerify + dNext, messageSize);
+ crcNew = crcOrig = XXH64_digest(&xxhNew);
+ FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
+
+ // prepare next message
+ iNext += messageSize;
+ rNext += messageSize;
+ dNext += messageSize;
+ messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
+ if (rNext + messageSize > ringBufferSize) rNext = 0;
+ if (dNext + messageSize > dBufferSize) dNext = 0;
+ }
+ }
+
}
printf("All unit tests completed succesfully \n");
generated by cgit v0.12 at 2024-11-07 12:12:30 (GMT)