From 20e969e5793aa6773593df8768d068a1ae13f746 Mon Sep 17 00:00:00 2001
From: Yann Collet
Date: Mon, 5 Feb 2018 15:18:00 -0800
Subject: fuzzer: added low address compression test
is expected to work on linux+gcc only.
---
lib/lz4hc.c | 9 +++++----
lib/lz4hc.h | 8 ++++----
tests/fuzzer.c | 46 +++++++++++++++++++++++++++++++++++++++++++++-
3 files changed, 54 insertions(+), 9 deletions(-)
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index cface81..79cf651 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -856,16 +856,17 @@ int LZ4_resetStreamStateHC(void* state, char* inputBuffer)
LZ4HC_CCtx_internal *ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */
LZ4HC_init(ctx, (const BYTE*)inputBuffer);
- ctx->inputBuffer = (BYTE*)inputBuffer;
+ ctx->inputBuffer = inputBuffer;
return 0;
}
-void* LZ4_createHC (char* inputBuffer)
+void* LZ4_createHC (const char* inputBuffer)
{
LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOCATOR(1, sizeof(LZ4_streamHC_t));
if (hc4 == NULL) return NULL; /* not enough memory */
LZ4HC_init (&hc4->internal_donotuse, (const BYTE*)inputBuffer);
- hc4->internal_donotuse.inputBuffer = (BYTE*)inputBuffer;
+ assert(sizeof(size_t) == sizeof(void*));
+ hc4->internal_donotuse.inputBuffer = (void*)(size_t)inputBuffer; /* ugly hack, circumvent -Wcast-qual */
return hc4;
}
@@ -889,5 +890,5 @@ char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
{
LZ4HC_CCtx_internal* const hc4 = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse;
int const dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB);
- return (char*)(hc4->inputBuffer + dictSize);
+ return (char*)(hc4->inputBuffer) + dictSize;
}
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index a7f77f9..7a25bee 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -148,7 +148,7 @@ typedef struct
const uint8_t* end; /* next block here to continue on current prefix */
const uint8_t* base; /* All index relative to this position */
const uint8_t* dictBase; /* alternate base for extDict */
- uint8_t* inputBuffer; /* deprecated */
+ void* inputBuffer; /* deprecated */
uint32_t dictLimit; /* below that point, need extDict */
uint32_t lowLimit; /* below that point, no more dict */
uint32_t nextToUpdate; /* index from which to continue dictionary update */
@@ -164,7 +164,7 @@ typedef struct
const unsigned char* end; /* next block here to continue on current prefix */
const unsigned char* base; /* All index relative to this position */
const unsigned char* dictBase; /* alternate base for extDict */
- unsigned char* inputBuffer; /* deprecated */
+ void* inputBuffer; /* deprecated */
unsigned int dictLimit; /* below that point, need extDict */
unsigned int lowLimit; /* below that point, no more dict */
unsigned int nextToUpdate; /* index from which to continue dictionary update */
@@ -206,8 +206,8 @@ LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_co
LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
-/* Deprecated Streaming functions using older model; should no longer be used */
-LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (char* inputBuffer);
+/* Deprecated Streaming functions; should no longer be used */
+LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (const char* inputBuffer);
LZ4_DEPRECATED("use LZ4_saveDictHC() instead") LZ4LIB_API char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") LZ4LIB_API int LZ4_freeHC (void* LZ4HC_Data);
LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index c134fe3..9415e94 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -240,6 +240,42 @@ _overflowError:
}
+#ifdef __unix__ /* is expected to be triggered on linux+gcc */
+
+#include /* mmap */
+
+static void* FUZ_createLowAddr(size_t size)
+{
+ void* const lowBuff = mmap((void*)(0x1000), size,
+ PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS,
+ -1, 0);
+ DISPLAYLEVEL(2, "generating low buffer at address %p \n", lowBuff);
+ return lowBuff;
+}
+
+static void FUZ_freeLowAddr(void* buffer, size_t size)
+{
+ if (munmap(buffer, size)) {
+ perror("fuzzer: freeing low address buffer");
+ abort();
+ }
+}
+
+#else
+
+static void* FUZ_createLowAddr(size_t size)
+{
+ return malloc(size);
+}
+
+static void FUZ_freeLowAddr(void* buffer, size_t size)
+{
+ (void)size;
+ free(buffer);
+}
+
+#endif
+
/*! FUZ_findDiff() :
* find the first different byte between buff1 and buff2.
* presumes buff1 != buff2.
@@ -266,6 +302,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
size_t const compressedBufferSize = LZ4_compressBound(FUZ_MAX_BLOCK_SIZE);
char* const compressedBuffer = (char*)malloc(compressedBufferSize);
char* const decodedBuffer = (char*)malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE);
+ size_t const labSize = 96 KB;
+ void* const lowAddrBuffer = FUZ_createLowAddr(labSize);
void* const stateLZ4 = malloc(LZ4_sizeofState());
void* const stateLZ4HC = malloc(LZ4_sizeofStateHC());
LZ4_stream_t LZ4dict;
@@ -306,7 +344,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
int const dictSize = MIN(dictSizeRand, blockStart);
int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1);
- char* const block = ((char*)CNBuffer) + blockStart;
+ const char* block = ((char*)CNBuffer) + blockStart;
const char* dict = block - dictSize;
int compressedSize, HCcompressedSize;
int blockContinueCompressedSize;
@@ -317,6 +355,11 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_displayUpdate(cycleNb);
/* Compression tests */
+ if ( ((FUZ_rand(&randState) & 63) == 2)
+ && ((size_t)blockSize < labSize) ) {
+ memcpy(lowAddrBuffer, block, blockSize);
+ block = lowAddrBuffer;
+ }
/* Test compression destSize */
FUZ_DISPLAYTEST;
@@ -705,6 +748,7 @@ _exit:
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
+ FUZ_freeLowAddr(lowAddrBuffer, labSize);
free(stateLZ4);
free(stateLZ4HC);
return result;
--
cgit v0.12
From c3f0ed28ffa66fd7e28ec3b6dbbe95eb0974bfef Mon Sep 17 00:00:00 2001
From: test4973
Date: Wed, 21 Mar 2018 07:14:13 -0700
Subject: added low address fuzzer tests
---
tests/Makefile | 3 +-
tests/fuzzer.c | 113 +++++++++++++++++++++++++++++++--------------------------
2 files changed, 63 insertions(+), 53 deletions(-)
diff --git a/tests/Makefile b/tests/Makefile
index ddc0d2e..34b8b24 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -35,7 +35,8 @@ PRGDIR := ../programs
TESTDIR := versionsTest
PYTHON ?= python3
-DEBUGFLAGS = -g -DLZ4_DEBUG=1
+DEBUGLEVEL?= 1
+DEBUGFLAGS = -g -DLZ4_DEBUG=$(DEBUGLEVEL)
CFLAGS ?= -O3 # can select custom optimization flags. For example : CFLAGS=-O2 make
CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 9415e94..2b9b926 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -34,9 +34,13 @@
#define LZ4_DISABLE_DEPRECATE_WARNINGS
+
/*-************************************
* Dependencies
**************************************/
+#ifdef __unix__ /* must be included before platform.h for MAP_ANONYMOUS */
+# include /* mmap */
+#endif
#include "platform.h" /* _CRT_SECURE_NO_WARNINGS */
#include "util.h" /* U32 */
#include
@@ -242,8 +246,6 @@ _overflowError:
#ifdef __unix__ /* is expected to be triggered on linux+gcc */
-#include /* mmap */
-
static void* FUZ_createLowAddr(size_t size)
{
void* const lowBuff = mmap((void*)(0x1000), size,
@@ -276,6 +278,7 @@ static void FUZ_freeLowAddr(void* buffer, size_t size)
#endif
+
/*! FUZ_findDiff() :
* find the first different byte between buff1 and buff2.
* presumes buff1 != buff2.
@@ -316,10 +319,18 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \
printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; }
-# define FUZ_DISPLAYTEST { testNb++; g_displayLevel>=4 ? printf("%2u\b\b", testNb), fflush(stdout) : 0; }
+# define FUZ_DISPLAYTEST(...) { \
+ testNb++; \
+ if (g_displayLevel>=4) { \
+ printf("\r%4u - %2u ", seed, testNb); \
+ printf(" " __VA_ARGS__); \
+ printf(" "); \
+ fflush(stdout); \
+ } }
/* init */
+ DISPLAYLEVEL(2, " g_displayLevel = %u \n", g_displayLevel);
if(!CNBuffer || !compressedBuffer || !decodedBuffer) {
DISPLAY("Not enough memory to start fuzzer tests");
goto _output_error;
@@ -362,7 +373,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
}
/* Test compression destSize */
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST("test LZ4_compress_destSize()");
{ int srcSize = blockSize;
int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7;
char endCheck = FUZ_rand(&randState) & 255;
@@ -377,7 +388,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
U32 const crcBase = XXH32(block, srcSize, 0);
char const canary = FUZ_rand(&randState) & 255;
FUZ_CHECKTEST((ret==0), "LZ4_compress_destSize() compression failed");
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
compressedSize = ret;
decodedBuffer[srcSize] = canary;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize);
@@ -393,7 +404,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
} }
/* Test compression HC destSize */
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST("test LZ4_compress_HC_destSize()");
{ int srcSize = blockSize;
int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7;
char const endCheck = FUZ_rand(&randState) & 255;
@@ -407,14 +418,12 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_HC_destSize() result larger than dst buffer !");
FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_HC_destSize() overwrite dst buffer !");
FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_HC_destSize() fed more than src buffer !");
- DISPLAYLEVEL(5, "LZ4_compress_HC_destSize(%i): destSize : %7i/%7i; content%7i/%7i ",
- compressionLevel, ret, targetSize, srcSize, blockSize);
if (targetSize>0) {
/* check correctness */
U32 const crcBase = XXH32(block, srcSize, 0);
char const canary = FUZ_rand(&randState) & 255;
FUZ_CHECKTEST((ret==0), "LZ4_compress_HC_destSize() compression failed");
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
compressedSize = ret;
decodedBuffer[srcSize] = canary;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize);
@@ -430,31 +439,31 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
} }
/* Test compression HC */
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST("test LZ4_compress_HC()");
ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
FUZ_CHECKTEST(ret==0, "LZ4_compress_HC() failed");
HCcompressedSize = ret;
/* Test compression HC using external state */
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC()");
ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed");
/* Test compression using external state */
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST("test LZ4_compress_fast_extState()");
ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed");
/* Test compression */
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST("test LZ4_compress_default()");
ret = LZ4_compress_default(block, compressedBuffer, blockSize, (int)compressedBufferSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress_default() failed");
compressedSize = ret;
/* Decompression tests */
- /* Test decoding with output size being exactly what's necessary => must work */
- FUZ_DISPLAYTEST;
+ /* Test decoding with output size exactly correct => must work */
+ FUZ_DISPLAYTEST("LZ4_decompress_fast() with exact output buffer");
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space");
FUZ_CHECKTEST(ret!=compressedSize, "LZ4_decompress_fast failed : did not fully read compressed data");
@@ -462,19 +471,19 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data");
/* Test decoding with one byte missing => must fail */
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST("LZ4_decompress_fast() with output buffer 1-byte too short");
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer");
/* Test decoding with one byte too much => must fail */
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");
/* Test decoding with output size exactly what's necessary => must work */
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space");
@@ -484,7 +493,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
// Test decoding with more than enough output size => must work
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
decodedBuffer[blockSize] = 0;
decodedBuffer[blockSize+1] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize+1);
@@ -496,14 +505,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
// Test decoding with output size being one byte too short => must fail
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe overrun specified output buffer size");
// Test decoding with output size being 10 bytes too short => must fail
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
if (blockSize>10) {
decodedBuffer[blockSize-10] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-10);
@@ -512,51 +521,51 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
}
// Test decoding with input size being one byte too short => must fail
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize);
// Test decoding with input size being one byte too large => must fail
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize+1, blockSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
// Test partial decoding with target output size being max/2 => must work
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize/2, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
// Test partial decoding with target output size being just below max => must work
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize-3, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
/* Test Compression with limited output size */
/* Test compression with output size being exactly what's necessary (should work) */
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST("test LZ4_compress_default() with output buffer just the right size");
ret = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress_default() failed despite sufficient space");
/* Test compression with output size being exactly what's necessary and external state (should work) */
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST("test LZ4_compress_fast_extState() with output buffer just the right size");
ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, compressedSize, 1);
FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed despite sufficient space");
/* Test HC compression with output size being exactly what's necessary (should work) */
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST("test LZ4_compress_HC() with output buffer just the right size");
ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel);
FUZ_CHECKTEST(ret==0, "LZ4_compress_HC() failed despite sufficient space");
/* Test HC compression with output size being exactly what's necessary (should work) */
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC() with output buffer just the right size");
ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel);
FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed despite sufficient space");
/* Test compression with missing bytes into output buffer => must fail */
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST("test LZ4_compress_default() with output buffer a bit too short");
{ int missingBytes = (FUZ_rand(&randState) % 0x3F) + 1;
if (missingBytes >= compressedSize) missingBytes = compressedSize-1;
missingBytes += !missingBytes; /* avoid special case missingBytes==0 */
@@ -567,7 +576,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
}
/* Test HC compression with missing bytes into output buffer => must fail */
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST("test LZ4_compress_HC() with output buffer a bit too short");
{ int missingBytes = (FUZ_rand(&randState) % 0x3F) + 1;
if (missingBytes >= HCcompressedSize) missingBytes = HCcompressedSize-1;
missingBytes += !missingBytes; /* avoid special case missingBytes==0 */
@@ -583,7 +592,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
/*-******************/
/* Compress using dictionary */
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary of size %i", dictSize);
{ LZ4_stream_t LZ4_stream;
LZ4_resetStream(&LZ4_stream);
LZ4_compress_fast_continue (&LZ4_stream, dict, compressedBuffer, dictSize, (int)compressedBufferSize, 1); /* Just to fill hash tables */
@@ -592,7 +601,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
}
/* Decompress with dictionary as prefix */
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST("test LZ4_decompress_fast_usingDict() with dictionary as prefix");
memcpy(decodedBuffer, dict, dictSize);
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer+dictSize, blockSize, decodedBuffer, dictSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
@@ -605,33 +614,33 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
}
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST("test LZ4_decompress_safe_usingDict()");
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
/* Compress using External dictionary */
- FUZ_DISPLAYTEST;
- dict -= (FUZ_rand(&randState) & 0xF) + 1; /* Separation, so it is an ExtDict */
+ FUZ_DISPLAYTEST("test LZ4_compress_fast_continue(), with non-contiguous dictionary");
+ dict -= (FUZ_rand(&randState) & 0xF) + 1; /* create space, so now dictionary is an ExtDict */
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
LZ4_loadDict(&LZ4dict, dict, dictSize);
blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue failed");
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary but with an output buffer too short by one byte");
LZ4_loadDict(&LZ4dict, dict, dictSize);
ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary loaded with LZ4_loadDict()");
LZ4_loadDict(&LZ4dict, dict, dictSize);
ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue should work : enough size available within output buffer");
/* Decompress with dictionary as external */
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
@@ -640,7 +649,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
@@ -648,19 +657,19 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
{ U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
if ((U32)blockSize > missingBytes) {
decodedBuffer[blockSize-missingBytes] = 0;
@@ -670,7 +679,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
} }
/* Compress HC using External dictionary */
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
LZ4_resetStreamHC (&LZ4dictHC, compressionLevel);
@@ -679,18 +688,18 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue failed");
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDict should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize);
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue should work : enough size available within output buffer");
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
@@ -701,10 +710,10 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
/* Compress HC continue destSize */
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
{ int const availableSpace = (FUZ_rand(&randState) % blockSize) + 5;
int consumedSize = blockSize;
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
LZ4_resetStreamHC (&LZ4dictHC, compressionLevel);
LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
blockContinueCompressedSize = LZ4_compress_HC_continue_destSize(&LZ4dictHC, block, compressedBuffer, &consumedSize, availableSpace);
@@ -713,7 +722,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(blockContinueCompressedSize > availableSpace, "LZ4_compress_HC_continue_destSize write overflow");
FUZ_CHECKTEST(consumedSize > blockSize, "LZ4_compress_HC_continue_destSize read overflow");
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
decodedBuffer[consumedSize] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, consumedSize, dict, dictSize);
FUZ_CHECKTEST(ret!=consumedSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
@@ -1118,13 +1127,13 @@ int main(int argc, const char** argv)
return FUZ_usage(programName);
case 'v': /* verbose mode */
- argument++;
g_displayLevel++;
+ argument++;
break;
case 'p': /* pause at the end */
- argument++;
use_pause=1;
+ argument++;
break;
case 'i':
--
cgit v0.12
From 606afdb16428347d0bf216b2c614d962018b3fc3 Mon Sep 17 00:00:00 2001
From: Kenji Chan
Date: Mon, 2 Apr 2018 10:52:45 +0800
Subject: added vs2017 projects
---
visual/VS2017/datagen/datagen.vcxproj | 173 ++++++++++++++++++++
visual/VS2017/frametest/frametest.vcxproj | 180 +++++++++++++++++++++
visual/VS2017/fullbench-dll/fullbench-dll.vcxproj | 184 ++++++++++++++++++++++
visual/VS2017/fullbench/fullbench.vcxproj | 180 +++++++++++++++++++++
visual/VS2017/fuzzer/fuzzer.vcxproj | 177 +++++++++++++++++++++
visual/VS2017/liblz4-dll/liblz4-dll.rc | 51 ++++++
visual/VS2017/liblz4-dll/liblz4-dll.vcxproj | 183 +++++++++++++++++++++
visual/VS2017/liblz4/liblz4.vcxproj | 179 +++++++++++++++++++++
visual/VS2017/lz4.sln | 98 ++++++++++++
9 files changed, 1405 insertions(+)
create mode 100644 visual/VS2017/datagen/datagen.vcxproj
create mode 100644 visual/VS2017/frametest/frametest.vcxproj
create mode 100644 visual/VS2017/fullbench-dll/fullbench-dll.vcxproj
create mode 100644 visual/VS2017/fullbench/fullbench.vcxproj
create mode 100644 visual/VS2017/fuzzer/fuzzer.vcxproj
create mode 100644 visual/VS2017/liblz4-dll/liblz4-dll.rc
create mode 100644 visual/VS2017/liblz4-dll/liblz4-dll.vcxproj
create mode 100644 visual/VS2017/liblz4/liblz4.vcxproj
create mode 100644 visual/VS2017/lz4.sln
diff --git a/visual/VS2017/datagen/datagen.vcxproj b/visual/VS2017/datagen/datagen.vcxproj
new file mode 100644
index 0000000..30e159e
--- /dev/null
+++ b/visual/VS2017/datagen/datagen.vcxproj
@@ -0,0 +1,173 @@
+
+
+
+
+ Debug
+ Win32
+
+
+ Debug
+ x64
+
+
+ Release
+ Win32
+
+
+ Release
+ x64
+
+
+
+ {D745AE2F-596A-403A-9B91-81A8C6779243}
+ Win32Proj
+ datagen
+ $(SolutionDir)bin\$(Platform)_$(Configuration)\
+ $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\
+
+
+
+ Application
+ true
+ Unicode
+ v141
+
+
+ Application
+ true
+ Unicode
+ v141
+
+
+ Application
+ false
+ Unicode
+ true
+ v141
+
+
+ Application
+ false
+ Unicode
+ true
+ v141
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ true
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+
+
+ true
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+ true
+
+
+ false
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+
+
+ false
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+ true
+
+
+
+
+
+ Level4
+ Disabled
+ WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
+ true
+ false
+ MultiThreadedDebug
+
+
+ Console
+ true
+
+
+
+
+
+
+ Level4
+ Disabled
+ WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
+ true
+ true
+ /analyze:stacksize295252 %(AdditionalOptions)
+ MultiThreadedDebug
+
+
+ Console
+ true
+
+
+
+
+ Level4
+
+
+ MaxSpeed
+ true
+ true
+ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
+ false
+ false
+ MultiThreaded
+
+
+ Console
+ true
+ true
+ true
+
+
+
+
+ Level4
+
+
+ MaxSpeed
+ true
+ true
+ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
+ false
+ true
+ /analyze:stacksize295252 %(AdditionalOptions)
+ MultiThreaded
+
+
+ Console
+ true
+ true
+ true
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/visual/VS2017/frametest/frametest.vcxproj b/visual/VS2017/frametest/frametest.vcxproj
new file mode 100644
index 0000000..a3a403d
--- /dev/null
+++ b/visual/VS2017/frametest/frametest.vcxproj
@@ -0,0 +1,180 @@
+
+
+
+
+ Debug
+ Win32
+
+
+ Debug
+ x64
+
+
+ Release
+ Win32
+
+
+ Release
+ x64
+
+
+
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}
+ Win32Proj
+ frametest
+ $(SolutionDir)bin\$(Platform)_$(Configuration)\
+ $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\
+
+
+
+ Application
+ true
+ Unicode
+ v141
+
+
+ Application
+ true
+ Unicode
+ v141
+
+
+ Application
+ false
+ Unicode
+ true
+ v141
+
+
+ Application
+ false
+ Unicode
+ true
+ v141
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ true
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+
+
+ true
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+ true
+
+
+ false
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+
+
+ false
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+ true
+
+
+
+
+
+ Level4
+ Disabled
+ WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
+ true
+ false
+ MultiThreadedDebug
+
+
+ Console
+ true
+
+
+
+
+
+
+ Level4
+ Disabled
+ WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
+ true
+ true
+ /analyze:stacksize295252 %(AdditionalOptions)
+ MultiThreadedDebug
+
+
+ Console
+ true
+
+
+
+
+ Level4
+
+
+ MaxSpeed
+ true
+ true
+ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
+ false
+ false
+ MultiThreaded
+
+
+ Console
+ true
+ true
+ true
+
+
+
+
+ Level4
+
+
+ MaxSpeed
+ true
+ true
+ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
+ false
+ true
+ /analyze:stacksize295252 %(AdditionalOptions)
+ MultiThreaded
+
+
+ Console
+ true
+ true
+ true
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/visual/VS2017/fullbench-dll/fullbench-dll.vcxproj b/visual/VS2017/fullbench-dll/fullbench-dll.vcxproj
new file mode 100644
index 0000000..d54a8d7
--- /dev/null
+++ b/visual/VS2017/fullbench-dll/fullbench-dll.vcxproj
@@ -0,0 +1,184 @@
+
+
+
+
+ Debug
+ Win32
+
+
+ Debug
+ x64
+
+
+ Release
+ Win32
+
+
+ Release
+ x64
+
+
+
+ {13992FD2-077E-4954-B065-A428198201A9}
+ Win32Proj
+ fullbench-dll
+ $(SolutionDir)bin\$(Platform)_$(Configuration)\
+ $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\
+
+
+
+ Application
+ true
+ Unicode
+ v141
+
+
+ Application
+ true
+ Unicode
+ v141
+
+
+ Application
+ false
+ Unicode
+ true
+ v141
+
+
+ Application
+ false
+ Unicode
+ true
+ v141
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ true
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+
+
+ true
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+ true
+
+
+ false
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+
+
+ false
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+ true
+
+
+
+
+
+ Level4
+ Disabled
+ WIN32;_DEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions)
+ true
+ false
+ MultiThreadedDebug
+
+
+ Console
+ true
+ $(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)
+ liblz4.lib;%(AdditionalDependencies)
+
+
+
+
+
+
+ Level4
+ Disabled
+ WIN32;_DEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions)
+ true
+ true
+ /analyze:stacksize295252 %(AdditionalOptions)
+ MultiThreadedDebug
+
+
+ Console
+ true
+ $(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)
+ liblz4.lib;%(AdditionalDependencies)
+
+
+
+
+ Level4
+
+
+ MaxSpeed
+ true
+ true
+ WIN32;NDEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions)
+ false
+ false
+ MultiThreaded
+
+
+ Console
+ true
+ true
+ true
+ $(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)
+ liblz4.lib;%(AdditionalDependencies)
+
+
+
+
+ Level4
+
+
+ MaxSpeed
+ true
+ true
+ WIN32;NDEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions)
+ false
+ true
+ /analyze:stacksize295252 %(AdditionalOptions)
+ MultiThreaded
+
+
+ Console
+ true
+ true
+ true
+ $(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)
+ liblz4.lib;%(AdditionalDependencies)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/visual/VS2017/fullbench/fullbench.vcxproj b/visual/VS2017/fullbench/fullbench.vcxproj
new file mode 100644
index 0000000..54c9743
--- /dev/null
+++ b/visual/VS2017/fullbench/fullbench.vcxproj
@@ -0,0 +1,180 @@
+
+
+
+
+ Debug
+ Win32
+
+
+ Debug
+ x64
+
+
+ Release
+ Win32
+
+
+ Release
+ x64
+
+
+
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}
+ Win32Proj
+ fullbench
+ $(SolutionDir)bin\$(Platform)_$(Configuration)\
+ $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\
+
+
+
+ Application
+ true
+ Unicode
+ v141
+
+
+ Application
+ true
+ Unicode
+ v141
+
+
+ Application
+ false
+ Unicode
+ true
+ v141
+
+
+ Application
+ false
+ Unicode
+ true
+ v141
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ true
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+
+
+ true
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+ true
+
+
+ false
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+
+
+ false
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+ true
+
+
+
+
+
+ Level4
+ Disabled
+ WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
+ true
+ false
+ MultiThreadedDebug
+
+
+ Console
+ true
+
+
+
+
+
+
+ Level4
+ Disabled
+ WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
+ true
+ true
+ /analyze:stacksize295252 %(AdditionalOptions)
+ MultiThreadedDebug
+
+
+ Console
+ true
+
+
+
+
+ Level4
+
+
+ MaxSpeed
+ true
+ true
+ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
+ false
+ false
+ MultiThreaded
+
+
+ Console
+ true
+ true
+ true
+
+
+
+
+ Level4
+
+
+ MaxSpeed
+ true
+ true
+ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
+ false
+ true
+ /analyze:stacksize295252 %(AdditionalOptions)
+ MultiThreaded
+
+
+ Console
+ true
+ true
+ true
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/visual/VS2017/fuzzer/fuzzer.vcxproj b/visual/VS2017/fuzzer/fuzzer.vcxproj
new file mode 100644
index 0000000..aa6fe42
--- /dev/null
+++ b/visual/VS2017/fuzzer/fuzzer.vcxproj
@@ -0,0 +1,177 @@
+
+
+
+
+ Debug
+ Win32
+
+
+ Debug
+ x64
+
+
+ Release
+ Win32
+
+
+ Release
+ x64
+
+
+
+ {18B9F1A7-9C66-4352-898B-30804DADE0FD}
+ Win32Proj
+ fuzzer
+ $(SolutionDir)bin\$(Platform)_$(Configuration)\
+ $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\
+
+
+
+ Application
+ true
+ Unicode
+ v141
+
+
+ Application
+ true
+ Unicode
+ v141
+
+
+ Application
+ false
+ Unicode
+ true
+ v141
+
+
+ Application
+ false
+ Unicode
+ true
+ v141
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ true
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+
+
+ true
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+ true
+
+
+ false
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+
+
+ false
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+ true
+
+
+
+
+
+ Level4
+ Disabled
+ WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
+ true
+ false
+ MultiThreadedDebug
+
+
+ Console
+ true
+
+
+
+
+
+
+ Level4
+ Disabled
+ WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
+ true
+ true
+ /analyze:stacksize295252 %(AdditionalOptions)
+ MultiThreadedDebug
+
+
+ Console
+ true
+
+
+
+
+ Level4
+
+
+ MaxSpeed
+ true
+ true
+ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
+ false
+ false
+ MultiThreaded
+
+
+ Console
+ true
+ true
+ true
+
+
+
+
+ Level4
+
+
+ MaxSpeed
+ true
+ true
+ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
+ false
+ true
+ /analyze:stacksize295252 %(AdditionalOptions)
+ MultiThreaded
+
+
+ Console
+ true
+ true
+ true
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/visual/VS2017/liblz4-dll/liblz4-dll.rc b/visual/VS2017/liblz4-dll/liblz4-dll.rc
new file mode 100644
index 0000000..b1871fe
--- /dev/null
+++ b/visual/VS2017/liblz4-dll/liblz4-dll.rc
@@ -0,0 +1,51 @@
+// Microsoft Visual C++ generated resource script.
+//
+
+#include "lz4.h" /* LZ4_VERSION_STRING */
+#define APSTUDIO_READONLY_SYMBOLS
+#include "verrsrc.h"
+#undef APSTUDIO_READONLY_SYMBOLS
+
+
+#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
+LANGUAGE 9, 1
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Version
+//
+
+VS_VERSION_INFO VERSIONINFO
+ FILEVERSION LZ4_VERSION_MAJOR,LZ4_VERSION_MINOR,LZ4_VERSION_RELEASE,0
+ PRODUCTVERSION LZ4_VERSION_MAJOR,LZ4_VERSION_MINOR,LZ4_VERSION_RELEASE,0
+ FILEFLAGSMASK VS_FFI_FILEFLAGSMASK
+#ifdef _DEBUG
+ FILEFLAGS VS_FF_DEBUG
+#else
+ FILEFLAGS 0x0L
+#endif
+ FILEOS VOS_NT_WINDOWS32
+ FILETYPE VFT_DLL
+ FILESUBTYPE VFT2_UNKNOWN
+BEGIN
+ BLOCK "StringFileInfo"
+ BEGIN
+ BLOCK "040904B0"
+ BEGIN
+ VALUE "CompanyName", "Yann Collet"
+ VALUE "FileDescription", "Extremely fast compression"
+ VALUE "FileVersion", LZ4_VERSION_STRING
+ VALUE "InternalName", "lz4.dll"
+ VALUE "LegalCopyright", "Copyright (C) 2013-2016, Yann Collet"
+ VALUE "OriginalFilename", "lz4.dll"
+ VALUE "ProductName", "LZ4"
+ VALUE "ProductVersion", LZ4_VERSION_STRING
+ END
+ END
+ BLOCK "VarFileInfo"
+ BEGIN
+ VALUE "Translation", 0x0409, 1200
+ END
+END
+
+#endif
diff --git a/visual/VS2017/liblz4-dll/liblz4-dll.vcxproj b/visual/VS2017/liblz4-dll/liblz4-dll.vcxproj
new file mode 100644
index 0000000..8e7ee3b
--- /dev/null
+++ b/visual/VS2017/liblz4-dll/liblz4-dll.vcxproj
@@ -0,0 +1,183 @@
+
+
+
+
+ Debug
+ Win32
+
+
+ Debug
+ x64
+
+
+ Release
+ Win32
+
+
+ Release
+ x64
+
+
+
+ {9800039D-4AAA-43A4-BB78-FEF6F4836927}
+ Win32Proj
+ liblz4-dll
+ $(SolutionDir)bin\$(Platform)_$(Configuration)\
+ $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\
+ liblz4-dll
+
+
+
+ DynamicLibrary
+ true
+ Unicode
+ v141
+
+
+ DynamicLibrary
+ true
+ Unicode
+ v141
+
+
+ DynamicLibrary
+ false
+ Unicode
+ true
+ v141
+
+
+ DynamicLibrary
+ false
+ Unicode
+ true
+ v141
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ true
+ liblz4
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+
+
+ true
+ liblz4
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+ true
+
+
+ false
+ liblz4
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+
+
+ false
+ liblz4
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+ true
+
+
+
+
+
+ Level4
+ Disabled
+ WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)
+ true
+ false
+ MultiThreadedDebug
+
+
+ true
+
+
+
+
+
+
+ Level4
+ Disabled
+ WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)
+ true
+ true
+ /analyze:stacksize295252 %(AdditionalOptions)
+ MultiThreadedDebug
+
+
+ true
+
+
+
+
+ Level4
+
+
+ MaxSpeed
+ true
+ true
+ WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)
+ false
+ false
+ MultiThreaded
+
+
+ true
+ true
+ true
+
+
+
+
+ Level4
+
+
+ MaxSpeed
+ true
+ true
+ WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)
+ false
+ true
+ /analyze:stacksize295252 %(AdditionalOptions)
+ MultiThreaded
+
+
+ true
+ true
+ true
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/visual/VS2017/liblz4/liblz4.vcxproj b/visual/VS2017/liblz4/liblz4.vcxproj
new file mode 100644
index 0000000..948f7db
--- /dev/null
+++ b/visual/VS2017/liblz4/liblz4.vcxproj
@@ -0,0 +1,179 @@
+
+
+
+
+ Debug
+ Win32
+
+
+ Debug
+ x64
+
+
+ Release
+ Win32
+
+
+ Release
+ x64
+
+
+
+ {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}
+ Win32Proj
+ liblz4
+ $(SolutionDir)bin\$(Platform)_$(Configuration)\
+ $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\
+
+
+
+ StaticLibrary
+ true
+ Unicode
+ v141
+
+
+ StaticLibrary
+ true
+ Unicode
+ v141
+
+
+ StaticLibrary
+ false
+ Unicode
+ true
+ v141
+
+
+ StaticLibrary
+ false
+ Unicode
+ true
+ v141
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ true
+ liblz4_static
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+
+
+ true
+ liblz4_static
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+ true
+
+
+ false
+ liblz4_static
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+
+
+ false
+ liblz4_static
+ $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);
+ true
+
+
+
+
+
+ Level4
+ Disabled
+ WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)
+ true
+ false
+ MultiThreadedDebug
+
+
+ true
+
+
+
+
+
+
+ Level4
+ Disabled
+ WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)
+ true
+ true
+ /analyze:stacksize295252 %(AdditionalOptions)
+ MultiThreadedDebug
+
+
+ true
+
+
+
+
+ Level4
+
+
+ MaxSpeed
+ true
+ true
+ WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)
+ false
+ false
+ MultiThreaded
+
+
+ true
+ true
+ true
+
+
+
+
+ Level4
+
+
+ MaxSpeed
+ true
+ true
+ WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)
+ false
+ true
+ /analyze:stacksize295252 %(AdditionalOptions)
+ MultiThreaded
+
+
+ true
+ true
+ true
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/visual/VS2017/lz4.sln b/visual/VS2017/lz4.sln
new file mode 100644
index 0000000..78f223b
--- /dev/null
+++ b/visual/VS2017/lz4.sln
@@ -0,0 +1,98 @@
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Express 2012 for Windows Desktop
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "lz4", "lz4\lz4.vcxproj", "{E30329AC-0057-4FE0-8FDA-7F650D398C4C}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "liblz4-dll", "liblz4-dll\liblz4-dll.vcxproj", "{9800039D-4AAA-43A4-BB78-FEF6F4836927}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "liblz4", "liblz4\liblz4.vcxproj", "{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcxproj", "{18B9F1A7-9C66-4352-898B-30804DADE0FD}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\fullbench.vcxproj", "{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "frametest", "frametest\frametest.vcxproj", "{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "datagen", "datagen\datagen.vcxproj", "{D745AE2F-596A-403A-9B91-81A8C6779243}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench-dll", "fullbench-dll\fullbench-dll.vcxproj", "{13992FD2-077E-4954-B065-A428198201A9}"
+ ProjectSection(ProjectDependencies) = postProject
+ {9800039D-4AAA-43A4-BB78-FEF6F4836927} = {9800039D-4AAA-43A4-BB78-FEF6F4836927}
+ EndProjectSection
+EndProject
+Global
+ GlobalSection(SolutionConfigurationPlatforms) = preSolution
+ Debug|Win32 = Debug|Win32
+ Debug|x64 = Debug|x64
+ Release|Win32 = Release|Win32
+ Release|x64 = Release|x64
+ EndGlobalSection
+ GlobalSection(ProjectConfigurationPlatforms) = postSolution
+ {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Win32.ActiveCfg = Debug|Win32
+ {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Win32.Build.0 = Debug|Win32
+ {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|x64.ActiveCfg = Debug|x64
+ {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|x64.Build.0 = Debug|x64
+ {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Win32.ActiveCfg = Release|Win32
+ {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Win32.Build.0 = Release|Win32
+ {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|x64.ActiveCfg = Release|x64
+ {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|x64.Build.0 = Release|x64
+ {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Debug|Win32.ActiveCfg = Debug|Win32
+ {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Debug|Win32.Build.0 = Debug|Win32
+ {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Debug|x64.ActiveCfg = Debug|x64
+ {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Debug|x64.Build.0 = Debug|x64
+ {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Release|Win32.ActiveCfg = Release|Win32
+ {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Release|Win32.Build.0 = Release|Win32
+ {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Release|x64.ActiveCfg = Release|x64
+ {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Release|x64.Build.0 = Release|x64
+ {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|Win32.ActiveCfg = Debug|Win32
+ {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|Win32.Build.0 = Debug|Win32
+ {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|x64.ActiveCfg = Debug|x64
+ {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|x64.Build.0 = Debug|x64
+ {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|Win32.ActiveCfg = Release|Win32
+ {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|Win32.Build.0 = Release|Win32
+ {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|x64.ActiveCfg = Release|x64
+ {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|x64.Build.0 = Release|x64
+ {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Win32.ActiveCfg = Debug|Win32
+ {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Win32.Build.0 = Debug|Win32
+ {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|x64.ActiveCfg = Debug|x64
+ {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|x64.Build.0 = Debug|x64
+ {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Win32.ActiveCfg = Release|Win32
+ {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Win32.Build.0 = Release|Win32
+ {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|x64.ActiveCfg = Release|x64
+ {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|x64.Build.0 = Release|x64
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|Win32.ActiveCfg = Debug|Win32
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|Win32.Build.0 = Debug|Win32
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|x64.ActiveCfg = Debug|x64
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|x64.Build.0 = Debug|x64
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|Win32.ActiveCfg = Release|Win32
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|Win32.Build.0 = Release|Win32
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|x64.ActiveCfg = Release|x64
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|x64.Build.0 = Release|x64
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|Win32.ActiveCfg = Debug|Win32
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|Win32.Build.0 = Debug|Win32
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|x64.ActiveCfg = Debug|x64
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|x64.Build.0 = Debug|x64
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|Win32.ActiveCfg = Release|Win32
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|Win32.Build.0 = Release|Win32
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|x64.ActiveCfg = Release|x64
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|x64.Build.0 = Release|x64
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|Win32.ActiveCfg = Debug|Win32
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|Win32.Build.0 = Debug|Win32
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|x64.ActiveCfg = Debug|x64
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|x64.Build.0 = Debug|x64
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|Win32.ActiveCfg = Release|Win32
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|Win32.Build.0 = Release|Win32
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|x64.ActiveCfg = Release|x64
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|x64.Build.0 = Release|x64
+ {13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.ActiveCfg = Debug|Win32
+ {13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.Build.0 = Debug|Win32
+ {13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.ActiveCfg = Debug|x64
+ {13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.Build.0 = Debug|x64
+ {13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.ActiveCfg = Release|Win32
+ {13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.Build.0 = Release|Win32
+ {13992FD2-077E-4954-B065-A428198201A9}.Release|x64.ActiveCfg = Release|x64
+ {13992FD2-077E-4954-B065-A428198201A9}.Release|x64.Build.0 = Release|x64
+ EndGlobalSection
+ GlobalSection(SolutionProperties) = preSolution
+ HideSolutionNode = FALSE
+ EndGlobalSection
+EndGlobal
--
cgit v0.12
From 6d931b6a93100fe2cb4dafc032ff486c814d1fed Mon Sep 17 00:00:00 2001
From: test4973
Date: Thu, 5 Apr 2018 12:40:33 -0700
Subject: fixed lz4 compression starting at small address
when using byU32 and byU16 modes
---
lib/lz4.c | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++++---
tests/fuzzer.c | 2 +-
2 files changed, 68 insertions(+), 4 deletions(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index 0fdbe5e..bcebc92 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -533,6 +533,20 @@ LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_
LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase);
}
+/* LZ4_getIndexOnHash() :
+ * Index of match position registered in hash table.
+ * hash position must be calculated by using base+index, or dictBase+index.
+ * Assumption 1 : only valid if tableType == byU32 or byU16.
+ * Assumption 2 : h is presumed valid (within limits of hash table)
+ */
+static U32 LZ4_getIndexOnHash(U32 h, const void* tableBase, tableType_t tableType)
+{
+ LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2);
+ if (tableType == byU32) { const U32* const hashTable = (const U32*) tableBase; assert(h < (1U << (LZ4_MEMORY_USAGE-2))); return hashTable[h]; }
+ if (tableType == byU16) { const U16* const hashTable = (const U16*) tableBase; assert(h < (1U << (LZ4_MEMORY_USAGE-1))); return hashTable[h]; }
+ assert(0); return 0; /* forbidden case */
+}
+
static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType_t tableType, const BYTE* srcBase)
{
if (tableType == byPtr) { const BYTE* const* hashTable = (const BYTE* const*) tableBase; return hashTable[h]; }
@@ -598,7 +612,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
{
const BYTE* ip = (const BYTE*) source;
- size_t currentOffset = cctx->currentOffset;
+ size_t const currentOffset = cctx->currentOffset;
const BYTE* base = (const BYTE*) source - currentOffset;
const BYTE* lowLimit;
@@ -650,7 +664,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
BYTE* token;
/* Find a match */
- { const BYTE* forwardIp = ip;
+ if (tableType == byPtr) {
+ const BYTE* forwardIp = ip;
unsigned step = 1;
unsigned searchMatchNb = acceleration << LZ4_skipTrigger;
do {
@@ -687,6 +702,54 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
} while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0)
|| ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
|| (LZ4_read32(match+refDelta) != LZ4_read32(ip)) );
+
+ } else { /* byU32, byU16 */
+
+ const BYTE* forwardIp = ip;
+ unsigned step = 1;
+ unsigned searchMatchNb = acceleration << LZ4_skipTrigger;
+ do {
+ U32 const h = forwardH;
+ U32 const matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType);
+ ip = forwardIp;
+ forwardIp += step;
+ step = (searchMatchNb++ >> LZ4_skipTrigger);
+
+ if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals;
+ assert(ip < mflimitPlusOne);
+
+ if (dictDirective == usingDictCtx) {
+ if (matchIndex < currentOffset) {
+ /* there was no match, try the dictionary */
+ match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
+ refDelta = dictDelta;
+ lowLimit = dictLowLimit;
+ } else {
+ match = base + matchIndex;
+ refDelta = 0;
+ lowLimit = (const BYTE*)source;
+ }
+ } else if (dictDirective==usingExtDict) {
+ if (matchIndex < currentOffset) {
+ match = dictBase + matchIndex;
+ refDelta = dictDelta;
+ lowLimit = dictLowLimit;
+ } else {
+ match = base + matchIndex;
+ refDelta = 0;
+ lowLimit = (const BYTE*)source;
+ }
+ } else { /* single continuous memory segment */
+ match = base + matchIndex;
+ refDelta = 0;
+ lowLimit = (const BYTE*)source;
+ }
+ forwardH = LZ4_hashPosition(forwardIp, tableType);
+ LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base);
+
+ } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0)
+ || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
+ || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) );
}
/* Catch up */
@@ -718,7 +781,8 @@ _next_match:
/* Encode MatchLength */
{ unsigned matchCode;
- if ((dictDirective==usingExtDict || dictDirective==usingDictCtx) && lowLimit==dictionary) {
+ if ( (dictDirective==usingExtDict || dictDirective==usingDictCtx)
+ && (lowLimit==dictionary) ) {
const BYTE* limit;
match += refDelta;
limit = ip + (dictEnd-match);
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 2b9b926..0b7d54e 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -322,7 +322,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
# define FUZ_DISPLAYTEST(...) { \
testNb++; \
if (g_displayLevel>=4) { \
- printf("\r%4u - %2u ", seed, testNb); \
+ printf("\r%4u - %2u ", cycleNb, testNb); \
printf(" " __VA_ARGS__); \
printf(" "); \
fflush(stdout); \
--
cgit v0.12
From 64a3e41acaf9e186937d32c9dd2dc104e5bc4a72 Mon Sep 17 00:00:00 2001
From: test4973
Date: Thu, 5 Apr 2018 16:38:43 -0700
Subject: changed LZ4_compress_generic() logic
to use indexes (U32) instead of Ptr.
byPtr is still present.
---
lib/lz4.c | 170 ++++++++++++++++++++++++++++++++++++--------------------------
1 file changed, 100 insertions(+), 70 deletions(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index bcebc92..4b219d2 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -516,6 +516,18 @@ LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tab
return LZ4_hash4(LZ4_read32(p), tableType);
}
+static void LZ4_putIndexOnHash(U32 index, U32 h, void* tableBase, tableType_t const tableType)
+{
+ switch (tableType)
+ {
+ default: /* fallthrough */
+ case clearedTable: /* fallthrough */
+ case byPtr: { /* illegal! */ assert(0); return; }
+ case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = index; return; }
+ case byU16: { U16* hashTable = (U16*) tableBase; assert(index < 65536); hashTable[h] = (U16)index; return; }
+ }
+}
+
static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase)
{
switch (tableType)
@@ -612,8 +624,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
{
const BYTE* ip = (const BYTE*) source;
- size_t const currentOffset = cctx->currentOffset;
- const BYTE* base = (const BYTE*) source - currentOffset;
+ size_t const startIndex = cctx->currentOffset;
+ const BYTE* base = (const BYTE*) source - startIndex;
const BYTE* lowLimit;
const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx;
@@ -622,7 +634,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
const U32 dictSize =
dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize;
- const BYTE* const lowRefLimit = (const BYTE*) source - dictSize;
+ int const maybe_ext_memSegment = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx);
+ U32 const prefixIdxLimit = startIndex - dictSize; /* used when dictDirective == dictSmall */
const BYTE* const dictEnd = dictionary + dictSize;
const BYTE* anchor = (const BYTE*) source;
const BYTE* const iend = ip + inputSize;
@@ -633,19 +646,20 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
* while a dictionary in the current context precedes the currentOffset */
const BYTE* dictBase = dictDirective == usingDictCtx ?
(const BYTE*) source - dictCtx->currentOffset :
- (const BYTE*) source - dictSize - currentOffset;
- const ptrdiff_t dictDelta = dictionary ? dictEnd - (const BYTE*) source : 0;
+ (const BYTE*) source - dictSize - startIndex;
const BYTE* dictLowLimit;
BYTE* op = (BYTE*) dest;
BYTE* const olimit = op + maxOutputSize;
+ U32 offset = 0;
ptrdiff_t retval = 0;
U32 forwardH;
/* Init conditions */
if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */
+ if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */
lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0);
dictLowLimit = dictionary ? dictionary : lowLimit;
@@ -659,7 +673,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
/* Main Loop */
for ( ; ; ) {
- ptrdiff_t refDelta = 0;
const BYTE* match;
BYTE* token;
@@ -678,82 +691,65 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
assert(ip < mflimitPlusOne);
match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base);
- if (dictDirective == usingDictCtx) {
- if (match < (const BYTE*)source) {
- /* there was no match, try the dictionary */
- match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
- refDelta = dictDelta;
- lowLimit = dictLowLimit;
- } else {
- refDelta = 0;
- lowLimit = (const BYTE*)source;
- }
- } else if (dictDirective==usingExtDict) {
- if (match < (const BYTE*)source) {
- refDelta = dictDelta;
- lowLimit = dictLowLimit;
- } else {
- refDelta = 0;
- lowLimit = (const BYTE*)source;
- } }
forwardH = LZ4_hashPosition(forwardIp, tableType);
LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base);
- } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0)
- || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
- || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) );
+ } while ( LZ4_read32(match) != LZ4_read32(ip) );
} else { /* byU32, byU16 */
const BYTE* forwardIp = ip;
- unsigned step = 1;
unsigned searchMatchNb = acceleration << LZ4_skipTrigger;
do {
U32 const h = forwardH;
- U32 const matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType);
+ U32 const current = forwardIp - base;
+ U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType);
+ assert(matchIndex <= current);
ip = forwardIp;
- forwardIp += step;
- step = (searchMatchNb++ >> LZ4_skipTrigger);
+ assert(searchMatchNb >= (1<> LZ4_skipTrigger);
if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals;
assert(ip < mflimitPlusOne);
if (dictDirective == usingDictCtx) {
- if (matchIndex < currentOffset) {
+ if (matchIndex < startIndex) {
/* there was no match, try the dictionary */
- match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
- refDelta = dictDelta;
+ matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32);
+ match = dictBase + matchIndex;
lowLimit = dictLowLimit;
} else {
match = base + matchIndex;
- refDelta = 0;
lowLimit = (const BYTE*)source;
}
} else if (dictDirective==usingExtDict) {
- if (matchIndex < currentOffset) {
+ if (matchIndex < startIndex) {
match = dictBase + matchIndex;
- refDelta = dictDelta;
lowLimit = dictLowLimit;
} else {
match = base + matchIndex;
- refDelta = 0;
lowLimit = (const BYTE*)source;
}
} else { /* single continuous memory segment */
match = base + matchIndex;
- refDelta = 0;
- lowLimit = (const BYTE*)source;
}
forwardH = LZ4_hashPosition(forwardIp, tableType);
- LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base);
+ LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
+
+ if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue; /* match outside of valid area */
+ if ((tableType != byU16) && (matchIndex+MAX_DISTANCE < current)) continue; /* too far */
+ if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE); /* too_far presumed impossible with byU16 */
- } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0)
- || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
- || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) );
+ if (LZ4_read32(match) == LZ4_read32(ip)) {
+ if (maybe_ext_memSegment) offset = current - matchIndex;
+ break; /* match found */
+ }
+
+ } while(1);
}
/* Catch up */
- while (((ip>anchor) & (match+refDelta > lowLimit)) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; }
+ while (((ip>anchor) & (match > lowLimit)) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; }
/* Encode Literals */
{ unsigned const litLength = (unsigned)(ip - anchor);
@@ -776,7 +772,13 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
_next_match:
/* Encode Offset */
- LZ4_writeLE16(op, (U16)(ip-match)); op+=2;
+ if (maybe_ext_memSegment) { /* static test */
+ assert(offset <= MAX_DISTANCE && offset > 0);
+ LZ4_writeLE16(op, (U16)offset); op+=2;
+ } else {
+ assert(ip-match <= MAX_DISTANCE);
+ LZ4_writeLE16(op, (U16)(ip - match)); op+=2;
+ }
/* Encode MatchLength */
{ unsigned matchCode;
@@ -784,7 +786,6 @@ _next_match:
if ( (dictDirective==usingExtDict || dictDirective==usingDictCtx)
&& (lowLimit==dictionary) ) {
const BYTE* limit;
- match += refDelta;
limit = ip + (dictEnd-match);
if (limit > matchlimit) limit = matchlimit;
matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit);
@@ -799,6 +800,8 @@ _next_match:
ip += MINMATCH + matchCode;
}
+ DEBUGLOG(2,"matchLength:%7u ", matchCode+MINMATCH);
+
if ( outputLimited && /* Check output buffer overflow */
(unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) )
goto _clean_up;
@@ -825,34 +828,61 @@ _next_match:
/* Fill table */
LZ4_putPosition(ip-2, cctx->hashTable, tableType, base);
+#if 1
/* Test next position */
- match = LZ4_getPosition(ip, cctx->hashTable, tableType, base);
- if (dictDirective == usingDictCtx) {
- if (match < (const BYTE*)source) {
- /* there was no match, try the dictionary */
- match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
- refDelta = dictDelta;
- lowLimit = dictLowLimit;
- } else {
- refDelta = 0;
- lowLimit = (const BYTE*)source;
+ if (tableType == byPtr) {
+
+ match = LZ4_getPosition(ip, cctx->hashTable, tableType, base);
+ LZ4_putPosition(ip, cctx->hashTable, tableType, base);
+ if ( (match+MAX_DISTANCE >= ip)
+ && (LZ4_read32(match) == LZ4_read32(ip)) )
+ { token=op++; *token=0; goto _next_match; }
+
+ } else { /* byU32, byU16 */
+
+ U32 const h = LZ4_hashPosition(ip, tableType);
+ U32 const current = (U32)(ip-base);
+ U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType);
+ assert(matchIndex < current);
+ if (dictDirective == usingDictCtx) {
+ if (match < (const BYTE*)source) {
+ /* there was no match, try the dictionary */
+ matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32);
+ match = dictBase + matchIndex;
+ } else {
+ match = base + matchIndex;
+ }
+ } else if (dictDirective==usingExtDict) {
+ if (match < (const BYTE*)source) {
+ match = dictBase + matchIndex;
+ } else {
+ match = base + matchIndex;
+ }
+ } else { /* single memory segment */
+ match = base + matchIndex;
}
- } else if (dictDirective==usingExtDict) {
- if (match < (const BYTE*)source) {
- refDelta = dictDelta;
- lowLimit = dictLowLimit;
- } else {
- refDelta = 0;
- lowLimit = (const BYTE*)source;
- } }
- LZ4_putPosition(ip, cctx->hashTable, tableType, base);
- if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1)
- && (match+MAX_DISTANCE>=ip)
- && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) )
- { token=op++; *token=0; goto _next_match; }
+ LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
+ if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1)
+ && ((tableType==byU16) ? 1 : (matchIndex+MAX_DISTANCE >= current))
+ && (LZ4_read32(match) == LZ4_read32(ip)) ) {
+ token=op++;
+ *token=0;
+ if (maybe_ext_memSegment)
+ offset = current - matchIndex;
+ goto _next_match;
+ }
+ }
/* Prepare next loop */
forwardH = LZ4_hashPosition(++ip, tableType);
+
+#else
+
+ /* Prepare next loop */
+ forwardH = LZ4_hashPosition(ip, tableType);
+
+#endif
+
}
_last_literals:
--
cgit v0.12
From f2a4d6ef37f653c21627274634d171af66126d5e Mon Sep 17 00:00:00 2001
From: test4973
Date: Thu, 5 Apr 2018 17:16:33 -0700
Subject: fixed immediate match search
---
lib/lz4.c | 12 +++++-------
1 file changed, 5 insertions(+), 7 deletions(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index 4b219d2..8d8c1e8 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -624,7 +624,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
{
const BYTE* ip = (const BYTE*) source;
- size_t const startIndex = cctx->currentOffset;
+ U32 const startIndex = cctx->currentOffset;
const BYTE* base = (const BYTE*) source - startIndex;
const BYTE* lowLimit;
@@ -645,8 +645,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
/* the dictCtx currentOffset is indexed on the start of the dictionary,
* while a dictionary in the current context precedes the currentOffset */
const BYTE* dictBase = dictDirective == usingDictCtx ?
- (const BYTE*) source - dictCtx->currentOffset :
- (const BYTE*) source - dictSize - startIndex;
+ dictionary + dictSize - dictCtx->currentOffset : /* is it possible that dictCtx->currentOffset != dictCtx->dictSize ? */
+ dictionary + dictSize - startIndex;
const BYTE* dictLowLimit;
BYTE* op = (BYTE*) dest;
@@ -800,8 +800,6 @@ _next_match:
ip += MINMATCH + matchCode;
}
- DEBUGLOG(2,"matchLength:%7u ", matchCode+MINMATCH);
-
if ( outputLimited && /* Check output buffer overflow */
(unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) )
goto _clean_up;
@@ -845,7 +843,7 @@ _next_match:
U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType);
assert(matchIndex < current);
if (dictDirective == usingDictCtx) {
- if (match < (const BYTE*)source) {
+ if (matchIndex < startIndex) {
/* there was no match, try the dictionary */
matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32);
match = dictBase + matchIndex;
@@ -853,7 +851,7 @@ _next_match:
match = base + matchIndex;
}
} else if (dictDirective==usingExtDict) {
- if (match < (const BYTE*)source) {
+ if (matchIndex < startIndex) {
match = dictBase + matchIndex;
} else {
match = base + matchIndex;
--
cgit v0.12
From b4be1e0a743f2200eaf1c13d322c925b64b872e2 Mon Sep 17 00:00:00 2001
From: test4973
Date: Thu, 5 Apr 2018 17:52:54 -0700
Subject: fixed byPtr match search
---
lib/lz4.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index 8d8c1e8..5791556 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -694,7 +694,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
forwardH = LZ4_hashPosition(forwardIp, tableType);
LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base);
- } while ( LZ4_read32(match) != LZ4_read32(ip) );
+ } while ( (match+MAX_DISTANCE < ip)
+ || (LZ4_read32(match) != LZ4_read32(ip)) );
} else { /* byU32, byU16 */
--
cgit v0.12
From f4e06e28e6d285f7f145798d7dfe1cbe71ae1efa Mon Sep 17 00:00:00 2001
From: test4973
Date: Thu, 5 Apr 2018 18:29:42 -0700
Subject: fixed byPtr mode
switch to byU32 when src address is < 64K
note : byPtr is still useful in 32-bits, as it's about ~10% faster
---
lib/lz4.c | 47 +++++++++++++++++++++++++----------------------
1 file changed, 25 insertions(+), 22 deletions(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index 5791556..4fb54f8 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -930,14 +930,14 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
if (inputSize < LZ4_64Klimit) {
return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration);
} else {
- const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+ const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > MAX_DISTANCE)) ? byPtr : byU32;
return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
}
} else {
if (inputSize < LZ4_64Klimit) {;
return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
} else {
- const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+ const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > MAX_DISTANCE)) ? byPtr : byU32;
return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
}
}
@@ -951,7 +951,7 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
* performance when the context reset would otherwise be a significant part of
* the cost of the compression, e.g., when the data to be compressed is small.
*/
-int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
+int LZ4_compress_fast_extState_noReset(void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration)
{
LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse;
if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
@@ -959,39 +959,39 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de
ctx->dictSize = 0;
ctx->dictCtx = NULL;
- if (maxOutputSize >= LZ4_compressBound(inputSize)) {
- if (inputSize < LZ4_64Klimit) {
+ if (dstCapacity >= LZ4_compressBound(srcSize)) {
+ if (srcSize < LZ4_64Klimit) {
const tableType_t tableType = byU16;
- LZ4_prepareTable(ctx, inputSize, tableType, noDict);
+ LZ4_prepareTable(ctx, srcSize, tableType, noDict);
if (ctx->currentOffset) {
- return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, dictSmall, acceleration);
+ return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, dictSmall, acceleration);
} else {
- return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
+ return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
}
} else {
- const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
- LZ4_prepareTable(ctx, inputSize, tableType, noDict);
+ const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
+ LZ4_prepareTable(ctx, srcSize, tableType, noDict);
if (ctx->currentOffset) {
ctx->currentOffset += 64 KB;
}
- return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
+ return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
}
} else {
- if (inputSize < LZ4_64Klimit) {
+ if (srcSize < LZ4_64Klimit) {
const tableType_t tableType = byU16;
- LZ4_prepareTable(ctx, inputSize, tableType, noDict);
+ LZ4_prepareTable(ctx, srcSize, tableType, noDict);
if (ctx->currentOffset) {
- return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration);
+ return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration);
} else {
- return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+ return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
}
} else {
- const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
- LZ4_prepareTable(ctx, inputSize, tableType, noDict);
+ const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
+ LZ4_prepareTable(ctx, srcSize, tableType, noDict);
if (ctx->currentOffset) {
ctx->currentOffset += 64 KB;
}
- return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+ return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
}
}
}
@@ -1098,6 +1098,8 @@ static int LZ4_compress_destSize_generic(
forwardH = LZ4_hashPosition(forwardIp, tableType);
LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base);
+ DEBUGLOG(2, "match:%p , ip:%p", match, ip);
+
} while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
|| (LZ4_read32(match) != LZ4_read32(ip)) );
}
@@ -1203,11 +1205,12 @@ static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src,
if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */
return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1);
} else {
- if (*srcSizePtr < LZ4_64Klimit)
+ if (*srcSizePtr < LZ4_64Klimit) {
return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16);
- else
- return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, sizeof(void*)==8 ? byU32 : byPtr);
- }
+ } else {
+ tableType_t const tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
+ return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, tableType);
+ } }
}
--
cgit v0.12
From 038a0d95bfe2d3a544aa2f5551998f6fd8bc0722 Mon Sep 17 00:00:00 2001
From: test4973
Date: Thu, 5 Apr 2018 18:39:22 -0700
Subject: added low-memory address test to travis
requires modification linux configuration (sudo)
---
.travis.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 466d55e..a446420 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,7 +15,7 @@ matrix:
- os: linux
sudo: false
- env: Ubu=12.04cont Cmd='make -C tests test-frametest test-fuzzer' COMPILER=cc
+ env: Ubu=12.04cont Cmd='sudo sysctl -w vm.mmap_min_addr="4096" && make -C tests test-frametest test-fuzzer' COMPILER=cc
- os: linux
sudo: false
@@ -59,7 +59,7 @@ matrix:
- libc6-dev-i386
- gcc-multilib
- - env: Ubu=14.04 Cmd='make -C tests test-frametest32 test-fuzzer32' COMPILER=cc
+ - env: Ubu=14.04 Cmd='sudo sysctl -w vm.mmap_min_addr="4096" && make -C tests test-frametest32 test-fuzzer32' COMPILER=cc
dist: trusty
sudo: required
addons:
--
cgit v0.12
From f9992fa37f1b0810c4d0a3e3e6a0eb4880168c57 Mon Sep 17 00:00:00 2001
From: test4973
Date: Thu, 5 Apr 2018 19:05:49 -0700
Subject: noticed a bug when re-using hash table
./fuzzer -vv -s4217 -t7518
---
lib/lz4.c | 7 ++++---
tests/fuzzer.c | 59 +++++++++++++++++++++++++++++-----------------------------
2 files changed, 33 insertions(+), 33 deletions(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index 4fb54f8..54336c7 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -593,6 +593,9 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
cctx->currentOffset = 0;
cctx->tableType = clearedTable;
+ } else {
+ DEBUGLOG(4, "Re-use hash table (no reset)");
+ //if (tableType == byU32) cctx->currentOffset += 64 KB;
}
}
/* If the current offset is zero, we will never look in the external
@@ -1098,8 +1101,6 @@ static int LZ4_compress_destSize_generic(
forwardH = LZ4_hashPosition(forwardIp, tableType);
LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base);
- DEBUGLOG(2, "match:%p , ip:%p", match, ip);
-
} while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
|| (LZ4_read32(match) != LZ4_read32(ip)) );
}
@@ -1371,7 +1372,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
LZ4_prepareTable(streamPtr, inputSize, tableType, usingDictCtx);
result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration);
}
- } else {
+ } else { /* no dictCtx */
LZ4_prepareTable(streamPtr, inputSize, tableType, usingExtDict);
if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 0b7d54e..8ffdb22 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -291,7 +291,7 @@ static void FUZ_findDiff(const void* buff1, const void* buff2)
const BYTE* const b2 = (const BYTE*)buff2;
size_t u = 0;
while (b1[u]==b2[u]) u++;
- DISPLAY("Wrong Byte at position %u \n", (unsigned)u);
+ DISPLAY("\nWrong Byte at position %u \n", (unsigned)u);
}
@@ -330,7 +330,6 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
/* init */
- DISPLAYLEVEL(2, " g_displayLevel = %u \n", g_displayLevel);
if(!CNBuffer || !compressedBuffer || !decodedBuffer) {
DISPLAY("Not enough memory to start fuzzer tests");
goto _output_error;
@@ -360,7 +359,6 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
int compressedSize, HCcompressedSize;
int blockContinueCompressedSize;
U32 const crcOrig = XXH32(block, blockSize, 0);
- U32 crcCheck;
int ret;
FUZ_displayUpdate(cycleNb);
@@ -467,8 +465,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space");
FUZ_CHECKTEST(ret!=compressedSize, "LZ4_decompress_fast failed : did not fully read compressed data");
- crcCheck = XXH32(decodedBuffer, blockSize, 0);
- FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data");
+ { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+ FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data");
+ }
/* Test decoding with one byte missing => must fail */
FUZ_DISPLAYTEST("LZ4_decompress_fast() with output buffer 1-byte too short");
@@ -489,8 +488,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space");
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
- crcCheck = XXH32(decodedBuffer, blockSize, 0);
- FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
+ { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+ FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
+ }
// Test decoding with more than enough output size => must work
FUZ_DISPLAYTEST();
@@ -501,8 +501,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data");
//FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe wrote more than (unknown) target size"); // well, is that an issue ?
FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ4_decompress_safe overrun specified output buffer size");
- crcCheck = XXH32(decodedBuffer, blockSize, 0);
- FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
+ { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+ FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
+ }
// Test decoding with output size being one byte too short => must fail
FUZ_DISPLAYTEST();
@@ -605,20 +606,17 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
memcpy(decodedBuffer, dict, dictSize);
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer+dictSize, blockSize, decodedBuffer, dictSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
- crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
- if (crcCheck!=crcOrig) {
- int i=0;
- while (block[i]==decodedBuffer[i]) i++;
- printf("Wrong Byte at position %i/%i\n", i, blockSize);
-
+ { U32 const crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
+ if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
+ FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
}
- FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
FUZ_DISPLAYTEST("test LZ4_decompress_safe_usingDict()");
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
- crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
- FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+ { U32 const crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
+ FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+ }
/* Compress using External dictionary */
FUZ_DISPLAYTEST("test LZ4_compress_fast_continue(), with non-contiguous dictionary");
@@ -640,22 +638,24 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue should work : enough size available within output buffer");
/* Decompress with dictionary as external */
- FUZ_DISPLAYTEST();
+ FUZ_DISPLAYTEST("test LZ4_decompress_fast_usingDict(): decoding %i bytes, dict(%p) of size %i", blockSize, dict, dictSize);
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
- crcCheck = XXH32(decodedBuffer, blockSize, 0);
- if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
- FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
+ { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+ if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
+ FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
+ }
FUZ_DISPLAYTEST();
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
- crcCheck = XXH32(decodedBuffer, blockSize, 0);
- FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+ { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+ FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+ }
FUZ_DISPLAYTEST();
decodedBuffer[blockSize-1] = 0;
@@ -704,10 +704,10 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
- crcCheck = XXH32(decodedBuffer, blockSize, 0);
- if (crcCheck!=crcOrig)
- FUZ_findDiff(block, decodedBuffer);
- FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+ { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+ if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
+ FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+ }
/* Compress HC continue destSize */
FUZ_DISPLAYTEST();
@@ -729,8 +729,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(decodedBuffer[consumedSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size")
{ U32 const crcSrc = XXH32(block, consumedSize, 0);
U32 const crcDst = XXH32(decodedBuffer, consumedSize, 0);
- if (crcSrc!=crcDst)
- FUZ_findDiff(block, decodedBuffer);
+ if (crcSrc!=crcDst) FUZ_findDiff(block, decodedBuffer);
FUZ_CHECKTEST(crcSrc!=crcDst, "LZ4_decompress_safe_usingDict corrupted decoded data");
}
}
--
cgit v0.12
From 133a50b780be302e726da772592e505d5b143f44 Mon Sep 17 00:00:00 2001
From: Yann Collet
Date: Fri, 6 Apr 2018 14:16:23 -0700
Subject: fixed DISPLAYUPDATE()
wrong comparison, which was always overflowing (hence was always true)
except when it was not (i386, reported by pmc)
in which case it would never show any information.
---
programs/lz4io.c | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/programs/lz4io.c b/programs/lz4io.c
index ca13316..6d0d0d0 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -94,9 +94,12 @@
static int g_displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
- if (((clock_t)(g_time - clock()) > refreshRate) || (g_displayLevel>=4)) \
- { g_time = clock(); DISPLAY(__VA_ARGS__); \
- if (g_displayLevel>=4) fflush(stderr); } }
+ if ( ((clock() - g_time) > refreshRate) \
+ || (g_displayLevel>=4) ) { \
+ g_time = clock(); \
+ DISPLAY(__VA_ARGS__); \
+ if (g_displayLevel>=4) fflush(stderr); \
+ } }
static const clock_t refreshRate = CLOCKS_PER_SEC / 6;
static clock_t g_time = 0;
--
cgit v0.12
From 5622c276e114c75f11f6171f9fe2295a8b626e89 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte"
Date: Fri, 6 Apr 2018 18:52:55 -0400
Subject: Return to Allowing Early Returns in LZ4_compress_generic()
Or: `goto` Considered Harmful
Or: https://xkcd.com/292/
---
lib/lz4.c | 38 +++++++++++++++++---------------------
1 file changed, 17 insertions(+), 21 deletions(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index 0fdbe5e..8743cb9 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -626,8 +626,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
BYTE* op = (BYTE*) dest;
BYTE* const olimit = op + maxOutputSize;
- ptrdiff_t retval = 0;
-
U32 forwardH;
/* Init conditions */
@@ -637,6 +635,19 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
dictLowLimit = dictionary ? dictionary : lowLimit;
if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */
+
+ /* Update context state */
+ if (dictDirective == usingDictCtx) {
+ /* Subsequent linked blocks can't use the dictionary. */
+ /* Instead, they use the block we just compressed. */
+ cctx->dictCtx = NULL;
+ cctx->dictSize = (U32)inputSize;
+ } else {
+ cctx->dictSize += (U32)inputSize;
+ }
+ cctx->currentOffset += (U32)inputSize;
+ cctx->tableType = tableType;
+
if (inputSize olimit)))
- goto _clean_up;
+ return 0;
if (litLength >= RUN_MASK) {
int len = (int)litLength-RUN_MASK;
*token = (RUN_MASK<>8) > olimit)) )
- goto _clean_up;
+ return 0;
if (matchCode >= ML_MASK) {
*token += ML_MASK;
matchCode -= ML_MASK;
@@ -796,7 +807,7 @@ _last_literals:
{ size_t const lastRun = (size_t)(iend - anchor);
if ( (outputLimited) && /* Check output buffer overflow */
((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) )
- goto _clean_up;
+ return 0;
if (lastRun >= RUN_MASK) {
size_t accumulator = lastRun - RUN_MASK;
*op++ = RUN_MASK << ML_BITS;
@@ -809,22 +820,7 @@ _last_literals:
op += lastRun;
}
- retval = (((char*)op)-dest);
-
-_clean_up:
- if (dictDirective == usingDictCtx) {
- /* Subsequent linked blocks can't use the dictionary. */
- /* Instead, they use the block we just compressed. */
- cctx->dictCtx = NULL;
- cctx->dictSize = (U32)inputSize;
- } else {
- cctx->dictSize += (U32)inputSize;
- }
- cctx->currentOffset += (U32)inputSize;
- cctx->tableType = tableType;
-
- /* End */
- return (int)retval;
+ return (int)(((char*)op) - dest);
}
--
cgit v0.12
From f88dc900553b3a029e9cd114800016f9364ffc7b Mon Sep 17 00:00:00 2001
From: "W. Felix Handte"
Date: Fri, 6 Apr 2018 16:52:29 -0400
Subject: Avoid Calling LZ4_prepareTable() in LZ4_compress_fast_continue()
---
lib/lz4.c | 34 ++++++++++++++--------------------
1 file changed, 14 insertions(+), 20 deletions(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index 8743cb9..4f353c5 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -574,11 +574,16 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
* indicates a miss. In that case, we need to bump the offset to something
* non-zero.
*/
- if (dictDirective == usingDictCtx &&
- tableType != byPtr &&
- cctx->currentOffset == 0)
- {
- cctx->currentOffset = 1;
+ if (cctx->currentOffset == 0) {
+ if (dictDirective == usingDictCtx) {
+ if (tableType == byU16) {
+ cctx->currentOffset = 1;
+ } else if (tableType == byU32) {
+ cctx->currentOffset = 64 KB;
+ }
+ }
+ } else if (tableType == byU32) {
+ cctx->currentOffset += 64 KB;
}
}
@@ -874,9 +879,6 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de
} else {
const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
LZ4_prepareTable(ctx, inputSize, tableType, noDict);
- if (ctx->currentOffset) {
- ctx->currentOffset += 64 KB;
- }
return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
}
} else {
@@ -891,9 +893,6 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de
} else {
const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
LZ4_prepareTable(ctx, inputSize, tableType, noDict);
- if (ctx->currentOffset) {
- ctx->currentOffset += 64 KB;
- }
return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
}
}
@@ -1168,31 +1167,28 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
{
LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse;
+ const tableType_t tableType = byU32;
const BYTE* p = (const BYTE*)dictionary;
const BYTE* const dictEnd = p + dictSize;
const BYTE* base;
DEBUGLOG(4, "LZ4_loadDict %p", LZ4_dict);
- if ((dict->initCheck)
- || (dict->tableType != byU32 && dict->tableType != clearedTable)
- || (dict->currentOffset > 1 GB)) /* Uninitialized structure, or reuse overflow */
- LZ4_resetStream(LZ4_dict);
+ LZ4_prepareTable(dict, 0, tableType, usingExtDict);
if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;
- dict->currentOffset += 64 KB;
base = p - dict->currentOffset;
dict->dictionary = p;
dict->dictSize = (U32)(dictEnd - p);
dict->currentOffset += dict->dictSize;
- dict->tableType = byU32;
+ dict->tableType = tableType;
if (dictSize < (int)HASH_UNIT) {
return 0;
}
while (p <= dictEnd-HASH_UNIT) {
- LZ4_putPosition(p, dict->hashTable, byU32, base);
+ LZ4_putPosition(p, dict->hashTable, tableType, base);
p+=3;
}
@@ -1244,7 +1240,6 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
/* prefix mode : source data follows dictionary */
if (dictEnd == (const BYTE*)source) {
- LZ4_prepareTable(streamPtr, inputSize, tableType, withPrefix64k);
if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration);
else
@@ -1272,7 +1267,6 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration);
}
} else {
- LZ4_prepareTable(streamPtr, inputSize, tableType, usingExtDict);
if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
} else {
--
cgit v0.12
From cf2f06a6c5046ef7d576bc8ad210c3f2efe1f401 Mon Sep 17 00:00:00 2001
From: test4973
Date: Mon, 9 Apr 2018 17:08:17 -0700
Subject: fixed minor conversion warning
ptr diff -> U32
---
lib/lz4.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index 3db37b0..a3c2860 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -717,9 +717,10 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
unsigned searchMatchNb = acceleration << LZ4_skipTrigger;
do {
U32 const h = forwardH;
- U32 const current = forwardIp - base;
+ U32 const current = (U32)(forwardIp - base);
U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType);
assert(matchIndex <= current);
+ assert(forwardIp - base < (ptrdiff_t)(2 GB - 1));
ip = forwardIp;
assert(searchMatchNb >= (1<> LZ4_skipTrigger);
--
cgit v0.12
From ad7e040384d1835c097b0f727aeaa3d598a401b6 Mon Sep 17 00:00:00 2001
From: test4973
Date: Mon, 9 Apr 2018 20:38:00 -0700
Subject: fix minor conversion warning
cast from void not implicit for C++
---
tests/fuzzer.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 8ffdb22..2e3ee92 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -367,7 +367,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
if ( ((FUZ_rand(&randState) & 63) == 2)
&& ((size_t)blockSize < labSize) ) {
memcpy(lowAddrBuffer, block, blockSize);
- block = lowAddrBuffer;
+ block = (const char*)lowAddrBuffer;
}
/* Test compression destSize */
--
cgit v0.12
From 59c7d951213bf8593438ae811549655d5a78d2fc Mon Sep 17 00:00:00 2001
From: "W. Felix Handte"
Date: Tue, 10 Apr 2018 13:12:30 -0400
Subject: Expose a Faster Stream Reset Function
---
lib/lz4.c | 53 ++++++++++++++++++++++++++++-------------------------
lib/lz4.h | 8 ++++++++
lib/lz4frame.c | 4 +---
3 files changed, 37 insertions(+), 28 deletions(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index 4f353c5..30146bb 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -546,12 +546,10 @@ LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, const void* tableBas
return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
}
-
LZ4_FORCE_INLINE void LZ4_prepareTable(
LZ4_stream_t_internal* const cctx,
const int inputSize,
- const tableType_t tableType,
- const dict_directive dictDirective) {
+ const tableType_t tableType) {
/* If the table hasn't been used, it's guaranteed to be zeroed out, and is
* therefore safe to use no matter what mode we're in. Otherwise, we figure
* out if it's safe to leave as is or whether it needs to be reset.
@@ -569,22 +567,23 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
cctx->tableType = clearedTable;
}
}
- /* If the current offset is zero, we will never look in the external
- * dictionary context, since there is no value a table entry can take that
- * indicates a miss. In that case, we need to bump the offset to something
- * non-zero.
+
+ /* Adding a gap, so all previous entries are > MAX_DISTANCE back, is faster
+ * than compressing without a gap. However, compressing with
+ * currentOffset == 0 is faster still, so we preserve that case.
*/
- if (cctx->currentOffset == 0) {
- if (dictDirective == usingDictCtx) {
- if (tableType == byU16) {
- cctx->currentOffset = 1;
- } else if (tableType == byU32) {
- cctx->currentOffset = 64 KB;
- }
- }
- } else if (tableType == byU32) {
+ if (cctx->currentOffset != 0 && tableType == byU32) {
cctx->currentOffset += 64 KB;
}
+
+ /* Finally, clear history */
+ cctx->dictCtx = NULL;
+ cctx->dictionary = NULL;
+ cctx->dictSize = 0;
+}
+
+void LZ4_resetStream_fast(LZ4_stream_t* const ctx) {
+ LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32);
}
/** LZ4_compress_generic() :
@@ -863,14 +862,11 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de
{
LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse;
if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
- ctx->dictionary = NULL;
- ctx->dictSize = 0;
- ctx->dictCtx = NULL;
if (maxOutputSize >= LZ4_compressBound(inputSize)) {
if (inputSize < LZ4_64Klimit) {
const tableType_t tableType = byU16;
- LZ4_prepareTable(ctx, inputSize, tableType, noDict);
+ LZ4_prepareTable(ctx, inputSize, tableType);
if (ctx->currentOffset) {
return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, dictSmall, acceleration);
} else {
@@ -878,13 +874,13 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de
}
} else {
const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
- LZ4_prepareTable(ctx, inputSize, tableType, noDict);
+ LZ4_prepareTable(ctx, inputSize, tableType);
return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
}
} else {
if (inputSize < LZ4_64Klimit) {
const tableType_t tableType = byU16;
- LZ4_prepareTable(ctx, inputSize, tableType, noDict);
+ LZ4_prepareTable(ctx, inputSize, tableType);
if (ctx->currentOffset) {
return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration);
} else {
@@ -892,7 +888,7 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de
}
} else {
const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
- LZ4_prepareTable(ctx, inputSize, tableType, noDict);
+ LZ4_prepareTable(ctx, inputSize, tableType);
return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
}
}
@@ -1174,7 +1170,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
DEBUGLOG(4, "LZ4_loadDict %p", LZ4_dict);
- LZ4_prepareTable(dict, 0, tableType, usingExtDict);
+ LZ4_prepareTable(dict, 0, tableType);
if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;
base = p - dict->currentOffset;
@@ -1263,7 +1259,14 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
} else {
- LZ4_prepareTable(streamPtr, inputSize, tableType, usingDictCtx);
+ /* If the current offset is zero, we will never look in the
+ * external dictionary context, since there is no value a table
+ * entry can take that indicate a miss. In that case, we need
+ * to bump the offset to something non-zero.
+ */
+ if (streamPtr->currentOffset == 0) {
+ streamPtr->currentOffset = 64 KB;
+ }
result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration);
}
} else {
diff --git a/lib/lz4.h b/lib/lz4.h
index 80f040f..9c1b694 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -366,6 +366,14 @@ LZ4_compress_fast_extState_noReset() :
LZ4LIB_API int LZ4_compress_fast_extState_noReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
+
+/*! LZ4_resetStream_fast() :
+ * An LZ4_stream_t structure can be allocated once and re-used multiple times.
+ * Use this function to start compressing a new stream.
+ */
+LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
+
+
/*-************************************
* Private definitions
**************************************
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index a080fa6..afa5af3 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -532,9 +532,7 @@ static void LZ4F_applyCDict(void* ctx,
if (level < LZ4HC_CLEVEL_MIN) {
LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t *)ctx)->internal_donotuse;
assert(!internal_ctx->initCheck);
- /* Clear any local dictionary */
- internal_ctx->dictionary = NULL;
- internal_ctx->dictSize = 0;
+ LZ4_resetStream_fast((LZ4_stream_t *)ctx);
/* Point to the dictionary context */
internal_ctx->dictCtx = cdict ? &(cdict->fastCtx->internal_donotuse) : NULL;
} else {
--
cgit v0.12
From c18bff933b18106ea000868ce13f37f0589cb58e Mon Sep 17 00:00:00 2001
From: "W. Felix Handte"
Date: Wed, 11 Apr 2018 15:12:34 -0400
Subject: Remove Extraneous Assignment (clearedTable == 0)
---
lib/lz4.c | 1 -
1 file changed, 1 deletion(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index 30146bb..f79ee2c 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1147,7 +1147,6 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
{
DEBUGLOG(5, "LZ4_resetStream %p", LZ4_stream);
MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
- LZ4_stream->internal_donotuse.tableType = clearedTable;
}
int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
--
cgit v0.12
From 21f0c9700b28301bdc8b49fcbe4ccdddf4cf5dea Mon Sep 17 00:00:00 2001
From: "W. Felix Handte"
Date: Wed, 11 Apr 2018 15:13:01 -0400
Subject: Rename _extState_noReset -> _extState_fastReset and Edit Comments
---
lib/lz4.c | 23 ++++++++++++-----------
lib/lz4.h | 43 ++++++++++++++++++++++++++++---------------
lib/lz4frame.c | 2 +-
3 files changed, 41 insertions(+), 27 deletions(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index f79ee2c..c2d1c32 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -582,10 +582,6 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
cctx->dictSize = 0;
}
-void LZ4_resetStream_fast(LZ4_stream_t* const ctx) {
- LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32);
-}
-
/** LZ4_compress_generic() :
inlined, to ensure branches are decided at compilation time */
LZ4_FORCE_INLINE int LZ4_compress_generic(
@@ -851,14 +847,15 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
}
/**
- * LZ4_compress_fast_extState_noReset is a variant of LZ4_compress_fast_extState
- * that can be used when the state is known to have already been initialized
- * (via LZ4_resetStream or an earlier call to LZ4_compress_fast_extState /
- * LZ4_compress_fast_extState_noReset). This can provide significantly better
- * performance when the context reset would otherwise be a significant part of
- * the cost of the compression, e.g., when the data to be compressed is small.
+ * LZ4_compress_fast_extState_fastReset() :
+ * A variant of LZ4_compress_fast_extState().
+ *
+ * Using this variant avoids an expensive initialization step. It is only safe
+ * to call if the state buffer is known to be correctly initialized already
+ * (see comment in lz4.h on LZ4_resetStream_fast() for a definition of
+ * "correctly initialized").
*/
-int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
+int LZ4_compress_fast_extState_fastReset(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
{
LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse;
if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
@@ -1149,6 +1146,10 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
}
+void LZ4_resetStream_fast(LZ4_stream_t* const ctx) {
+ LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32);
+}
+
int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
{
if (!LZ4_stream) return 0; /* support free on NULL */
diff --git a/lib/lz4.h b/lib/lz4.h
index 9c1b694..427beaf 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -353,26 +353,39 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or
* Their signatures may change.
**************************************/
-/*!
-LZ4_compress_fast_extState_noReset() :
- A variant of LZ4_compress_fast_extState().
-
- Use the _noReset variant if LZ4_resetStream() was called on the state
- buffer before being used for the first time (calls to both extState
- functions leave the state in a safe state, so zeroing is not required
- between calls). Otherwise, using the plain _extState requires LZ4 to
- reinitialize the state internally for every call.
-*/
-LZ4LIB_API int LZ4_compress_fast_extState_noReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
-
-
/*! LZ4_resetStream_fast() :
- * An LZ4_stream_t structure can be allocated once and re-used multiple times.
- * Use this function to start compressing a new stream.
+ * When an LZ4_stream_t is known to be in a internally coherent state,
+ * it can often be prepared for a new compression with almost no work, only
+ * sometimes falling back to the full, expensive reset that is always required
+ * when the stream is in an indeterminate state (i.e., the reset performed by
+ * LZ4_resetStream()).
+ *
+ * LZ4_streams are guaranteed to be in a valid state when:
+ * - returned from LZ4_createStream()
+ * - reset by LZ4_resetStream()
+ * - memset(stream, 0, sizeof(LZ4_stream_t))
+ * - the stream was in a valid state and was reset by LZ4_resetStream_fast()
+ * - the stream was in a valid state and was then used in any compression call
+ * that returned success
+ * - the stream was in an indeterminate state and was used in a compression
+ * call that fully reset the state (LZ4_compress_fast_extState()) and that
+ * returned success
*/
LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
+/*! LZ4_compress_fast_extState_fastReset() :
+ * A variant of LZ4_compress_fast_extState().
+ *
+ * Using this variant avoids an expensive initialization step. It is only safe
+ * to call if the state buffer is known to be correctly initialized already
+ * (see above comment on LZ4_resetStream_fast() for a definition of "correctly
+ * initialized"). From a high level, the difference is that this function
+ * initializes the provided state with a call to LZ4_resetStream_fast() while
+ * LZ4_compress_fast_extState() starts with a call to LZ4_resetStream().
+ */
+LZ4LIB_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
+
/*-************************************
* Private definitions
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index afa5af3..5e11ba2 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -714,7 +714,7 @@ static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize
if (cdict) {
return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
} else {
- return LZ4_compress_fast_extState_noReset(ctx, src, dst, srcSize, dstCapacity, acceleration);
+ return LZ4_compress_fast_extState_fastReset(ctx, src, dst, srcSize, dstCapacity, acceleration);
}
}
--
cgit v0.12
From afa52c9b95dd1f83d80cd9a783a408822700e2d3 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte"
Date: Wed, 11 Apr 2018 16:04:24 -0400
Subject: Expose dictCtx Functionality in LZ4
---
lib/lz4.c | 4 ++++
lib/lz4.h | 28 ++++++++++++++++++++++++++++
lib/lz4frame.c | 3 +--
3 files changed, 33 insertions(+), 2 deletions(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index c2d1c32..cee85ec 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1191,6 +1191,10 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
return dict->dictSize;
}
+void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream) {
+ working_stream->internal_donotuse.dictCtx = dictionary_stream != NULL ? &(dictionary_stream->internal_donotuse) : NULL;
+}
+
static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src)
{
diff --git a/lib/lz4.h b/lib/lz4.h
index 427beaf..f90120e 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -386,6 +386,34 @@ LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
*/
LZ4LIB_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
+/*! LZ4_attach_dictionary() :
+ * This is an experimental API that allows for the efficient use of a
+ * static dictionary many times.
+ *
+ * Rather than re-loading the dictionary buffer into a working context before
+ * each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a
+ * working LZ4_stream_t, this function introduces a no-copy setup mechanism,
+ * in which the working stream references the dictionary stream in-place.
+ *
+ * Several assumptions are made about the state of the dictionary stream.
+ * Currently, only streams which have been prepared by LZ4_loadDict() should
+ * be expected to work.
+ *
+ * Alternatively, the provided dictionary stream pointer may be NULL, in which
+ * case any existing dictionary stream is unset.
+ *
+ * If a dictionary is provided, it replaces any pre-existing stream history.
+ * The dictionary contents are the only history that can be referenced and
+ * logically immediately precede the data compressed in the first subsequent
+ * compression call.
+ *
+ * The dictionary will only remain attached to the working stream through the
+ * first compression call, at the end of which it is cleared. The dictionary
+ * stream (and source buffer) must remain in-place / accessible / unchanged
+ * through the completion of the first compression call on the stream.
+ */
+LZ4LIB_API void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream);
+
/*-************************************
* Private definitions
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 5e11ba2..d973b5f 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -533,8 +533,7 @@ static void LZ4F_applyCDict(void* ctx,
LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t *)ctx)->internal_donotuse;
assert(!internal_ctx->initCheck);
LZ4_resetStream_fast((LZ4_stream_t *)ctx);
- /* Point to the dictionary context */
- internal_ctx->dictCtx = cdict ? &(cdict->fastCtx->internal_donotuse) : NULL;
+ LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
} else {
if (cdict) {
memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx));
--
cgit v0.12
From 3a0c571272731166774405e1f96b3161827ca09f Mon Sep 17 00:00:00 2001
From: "W. Felix Handte"
Date: Wed, 11 Apr 2018 16:31:52 -0400
Subject: Add a LZ4_STATIC_LINKING_ONLY Macro to Guard Experimental APIs
---
lib/lz4.c | 1 +
lib/lz4.h | 2 ++
lib/lz4frame.c | 1 +
3 files changed, 4 insertions(+)
diff --git a/lib/lz4.c b/lib/lz4.c
index cee85ec..9e6b1e2 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -89,6 +89,7 @@
/*-************************************
* Dependency
**************************************/
+#define LZ4_STATIC_LINKING_ONLY
#include "lz4.h"
/* see also "memory routines" below */
diff --git a/lib/lz4.h b/lib/lz4.h
index f90120e..0dfa19e 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -353,6 +353,7 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or
* Their signatures may change.
**************************************/
+#ifdef LZ4_STATIC_LINKING_ONLY
/*! LZ4_resetStream_fast() :
* When an LZ4_stream_t is known to be in a internally coherent state,
@@ -414,6 +415,7 @@ LZ4LIB_API int LZ4_compress_fast_extState_fastReset (void* state, const char* sr
*/
LZ4LIB_API void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream);
+#endif
/*-************************************
* Private definitions
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index d973b5f..7608d90 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -74,6 +74,7 @@ You can contact the author at :
* Includes
**************************************/
#include "lz4frame_static.h"
+#define LZ4_STATIC_LINKING_ONLY
#include "lz4.h"
#define LZ4_HC_STATIC_LINKING_ONLY
#include "lz4hc.h"
--
cgit v0.12
From 51a56c47c0d34b6255bdd8e7b33f0ccb44b80351 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte"
Date: Wed, 11 Apr 2018 16:55:12 -0400
Subject: Minor Fixes
---
lib/lz4.c | 22 +++++++++++++---------
lib/lz4frame.c | 2 --
2 files changed, 13 insertions(+), 11 deletions(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index 9e6b1e2..111085a 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1193,7 +1193,19 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
}
void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream) {
- working_stream->internal_donotuse.dictCtx = dictionary_stream != NULL ? &(dictionary_stream->internal_donotuse) : NULL;
+ if (dictionary_stream != NULL) {
+ /* If the current offset is zero, we will never look in the
+ * external dictionary context, since there is no value a table
+ * entry can take that indicate a miss. In that case, we need
+ * to bump the offset to something non-zero.
+ */
+ if (working_stream->internal_donotuse.currentOffset == 0) {
+ working_stream->internal_donotuse.currentOffset = 64 KB;
+ }
+ working_stream->internal_donotuse.dictCtx = &(dictionary_stream->internal_donotuse);
+ } else {
+ working_stream->internal_donotuse.dictCtx = NULL;
+ }
}
@@ -1264,14 +1276,6 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
} else {
- /* If the current offset is zero, we will never look in the
- * external dictionary context, since there is no value a table
- * entry can take that indicate a miss. In that case, we need
- * to bump the offset to something non-zero.
- */
- if (streamPtr->currentOffset == 0) {
- streamPtr->currentOffset = 64 KB;
- }
result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration);
}
} else {
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 7608d90..507e4fe 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -531,8 +531,6 @@ static void LZ4F_applyCDict(void* ctx,
const LZ4F_CDict* cdict,
int level) {
if (level < LZ4HC_CLEVEL_MIN) {
- LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t *)ctx)->internal_donotuse;
- assert(!internal_ctx->initCheck);
LZ4_resetStream_fast((LZ4_stream_t *)ctx);
LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
} else {
--
cgit v0.12
From 056ea63215800c9c16e81db92eacc9a9c87548ae Mon Sep 17 00:00:00 2001
From: "W. Felix Handte"
Date: Wed, 11 Apr 2018 18:42:09 -0400
Subject: Fix Silly Warning (const-ness in declaration has no effect on value
types!)
---
lib/lz4.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index 111085a..4b0efb1 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1147,7 +1147,7 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
}
-void LZ4_resetStream_fast(LZ4_stream_t* const ctx) {
+void LZ4_resetStream_fast(LZ4_stream_t* ctx) {
LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32);
}
--
cgit v0.12
From 7b3cd10579fdc3d62bbf5e00f07ca19068a4403b Mon Sep 17 00:00:00 2001
From: Yann Collet
Date: Wed, 11 Apr 2018 16:31:43 -0700
Subject: reduced test time on circle-ci
---
Makefile | 4 ++--
circle.yml | 14 +++++++-------
tests/Makefile | 13 ++++++-------
3 files changed, 15 insertions(+), 16 deletions(-)
diff --git a/Makefile b/Makefile
index 3a95808..86613fd 100644
--- a/Makefile
+++ b/Makefile
@@ -143,10 +143,10 @@ clangtest-native: clean
@CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) -C $(TESTDIR) native CC=clang
usan: clean
- CC=clang CFLAGS="-O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T1mn" NB_LOOPS=-i1
+ CC=clang CFLAGS="-O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T30s" NB_LOOPS=-i1
usan32: clean
- CFLAGS="-m32 -O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T1mn" NB_LOOPS=-i1
+ CFLAGS="-m32 -O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T30s" NB_LOOPS=-i1
staticAnalyze: clean
CFLAGS=-g scan-build --status-bugs -v $(MAKE) all
diff --git a/circle.yml b/circle.yml
index 9a0d1ec..aeb52e9 100644
--- a/circle.yml
+++ b/circle.yml
@@ -11,10 +11,10 @@ test:
- clang -v; make clangtest && make clean
- g++ -v; make gpptest && make clean
- gcc -v; make c_standards && make clean
- - gcc-5 -v; make -C tests test-lz4 CC=gcc-5 MOREFLAGS=-Werror && make clean
- - gcc-5 -v; make -C tests test-lz4c32 CC=gcc-5 MOREFLAGS="-I/usr/include/x86_64-linux-gnu -Werror" && make clean
- - gcc-6 -v; make c_standards CC=gcc-6 && make clean
- - gcc-6 -v; make -C tests test-lz4 CC=gcc-6 MOREFLAGS=-Werror && make clean
+ - gcc-5 -v; CC=gcc-5 CFLAGS="-O2 -Werror" make check && make clean
+ - gcc-5 -v; CC=gcc-5 CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean
+ - gcc-6 -v; CC=gcc-6 make c_standards && make clean
+ - gcc-6 -v; CC=gcc-6 MOREFLAGS="-O2 -Werror" make check && make clean
# Shorter tests
- make cmake && make clean
- make -C tests test-lz4
@@ -22,11 +22,11 @@ test:
- make -C tests test-frametest
- make -C tests test-fullbench
- make -C tests test-fuzzer && make clean
- - make -C lib all && make clean
- - pyenv global 3.4.4; CFLAGS=-I/usr/include/x86_64-linux-gnu make versionsTest && make clean
+ - make -C lib all && make clean
+ - pyenv global 3.4.4; CPPFLAGS=-I/usr/include/x86_64-linux-gnu make versionsTest && make clean
- make travis-install && make clean
# Longer tests
- - gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean
+ - gcc -v; CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean
- make usan && make clean
- clang -v; make staticAnalyze && make clean
# Valgrind tests
diff --git a/tests/Makefile b/tests/Makefile
index 77f5d02..d11b7fe 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -56,7 +56,7 @@ LZ4 := $(PRGDIR)/lz4$(EXT)
# Default test parameters
TEST_FILES := COPYING
-FUZZER_TIME := -T3mn
+FUZZER_TIME := -T90s
NB_LOOPS ?= -i1
@@ -393,13 +393,12 @@ test-mem: lz4 datagen fuzzer frametest fullbench
./datagen -g16KB -s2 > ftmdg16K2
./datagen -g16KB -s3 > ftmdg16K3
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --force --multiple ftmdg16K ftmdg16K2 ftmdg16K3
- ./datagen -g16MB > ftmdg16M
- valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f ftmdg16M ftmdg16K2
+ ./datagen -g7MB > ftmdg7M
+ valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f ftmdg7M ftmdg16K2
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -t ftmdg16K2
- valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 ftmdg16M
- valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 ftmdg16M ftmdg16K2
- ./datagen -g256MB > ftmdg256M
- valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq ftmdg256M $(VOID)
+ valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 ftmdg7M
+ valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 ftmdg7M ftmdg16K2
+ valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq ftmdg7M $(VOID)
$(RM) ftm*
valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i64 -t1
valgrind --leak-check=yes --error-exitcode=1 ./frametest -i256
--
cgit v0.12
From ca388790e689ddb09e5e5f7f86a5bdb17dee67d7 Mon Sep 17 00:00:00 2001
From: Yann Collet
Date: Wed, 11 Apr 2018 16:41:25 -0700
Subject: allow system-defined CPPFLAGS in /tests
---
tests/Makefile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/Makefile b/tests/Makefile
index d11b7fe..25f00b8 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -39,7 +39,7 @@ CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
-Wpointer-arith -Wstrict-aliasing=1
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
-CPPFLAGS:= -I$(LZ4DIR) -I$(PRGDIR) -DXXH_NAMESPACE=LZ4_
+CPPFLAGS+= -I$(LZ4DIR) -I$(PRGDIR) -DXXH_NAMESPACE=LZ4_
FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
--
cgit v0.12
From 1838803948ceaf211b9e79fb405ef9b7340762ce Mon Sep 17 00:00:00 2001
From: test4973
Date: Wed, 11 Apr 2018 16:49:40 -0700
Subject: fixed LZ4_compress_fast_extState_fastReset()
---
lib/lz4.c | 15 +++++++--------
1 file changed, 7 insertions(+), 8 deletions(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index d79b2cc..d564ddc 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -953,7 +953,7 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
* (see comment in lz4.h on LZ4_resetStream_fast() for a definition of
* "correctly initialized").
*/
-int LZ4_compress_fast_extState_fastReset(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
+int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration)
{
LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse;
if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
@@ -961,8 +961,7 @@ int LZ4_compress_fast_extState_fastReset(void* state, const char* source, char*
if (dstCapacity >= LZ4_compressBound(srcSize)) {
if (srcSize < LZ4_64Klimit) {
const tableType_t tableType = byU16;
- LZ4_prepareTable(ctx, inputSize, tableType);
-LZ4_prepareTable>>> dev
+ LZ4_prepareTable(ctx, srcSize, tableType);
if (ctx->currentOffset) {
return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, dictSmall, acceleration);
} else {
@@ -970,13 +969,13 @@ LZ4_prepareTable>>> dev
}
} else {
const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
- LZ4_prepareTable(ctx, inputSize, tableType);
- return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
+ LZ4_prepareTable(ctx, srcSize, tableType);
+ return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
}
} else {
if (srcSize < LZ4_64Klimit) {
const tableType_t tableType = byU16;
- LZ4_prepareTable(ctx, inputSize, tableType);
+ LZ4_prepareTable(ctx, srcSize, tableType);
if (ctx->currentOffset) {
return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration);
} else {
@@ -984,8 +983,8 @@ LZ4_prepareTable>>> dev
}
} else {
const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
- LZ4_prepareTable(ctx, inputSize, tableType);
- return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+ LZ4_prepareTable(ctx, srcSize, tableType);
+ return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
}
}
}
--
cgit v0.12
From d6d9bccd537e307cc0bf3f8475801dea79e97367 Mon Sep 17 00:00:00 2001
From: Yann Collet
Date: Thu, 12 Apr 2018 06:47:27 -0700
Subject: modified versionsTest
to use MOREFLAGS rather CPPFLAGS
as some older versions of LZ4 overwrite CPPFLAGS environment variable.
---
circle.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/circle.yml b/circle.yml
index aeb52e9..fa37590 100644
--- a/circle.yml
+++ b/circle.yml
@@ -23,7 +23,7 @@ test:
- make -C tests test-fullbench
- make -C tests test-fuzzer && make clean
- make -C lib all && make clean
- - pyenv global 3.4.4; CPPFLAGS=-I/usr/include/x86_64-linux-gnu make versionsTest && make clean
+ - pyenv global 3.4.4; make versionsTest MOREFLAGS=-I/usr/include/x86_64-linux-gnu && make clean
- make travis-install && make clean
# Longer tests
- gcc -v; CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean
--
cgit v0.12
From 8af32ce6f7109ecf5cd7d73527e0aba3a63b55e5 Mon Sep 17 00:00:00 2001
From: test4973
Date: Thu, 12 Apr 2018 07:25:40 -0700
Subject: modified a few traces for debug
---
lib/lz4.c | 5 +++--
lib/lz4hc.c | 7 +++----
tests/fuzzer.c | 4 +++-
3 files changed, 9 insertions(+), 7 deletions(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index d564ddc..e4a68cd 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -594,7 +594,6 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
cctx->tableType = clearedTable;
} else {
DEBUGLOG(4, "Re-use hash table (no reset)");
- //if (tableType == byU32) cctx->currentOffset += 64 KB;
}
}
@@ -602,9 +601,11 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
* than compressing without a gap. However, compressing with
* currentOffset == 0 is faster still, so we preserve that case.
*/
+ DEBUGLOG(2, "tableType=%u, currentOffset=%u", cctx->tableType, cctx->currentOffset);
if (cctx->currentOffset != 0 && tableType == byU32) {
cctx->currentOffset += 64 KB;
}
+ DEBUGLOG(2, "currentOffset: %u", cctx->currentOffset);
/* Finally, clear history */
cctx->dictCtx = NULL;
@@ -1267,7 +1268,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
const BYTE* const dictEnd = p + dictSize;
const BYTE* base;
- DEBUGLOG(4, "LZ4_loadDict %p", LZ4_dict);
+ DEBUGLOG(4, "LZ4_loadDict (%p into %p)", dictionary, LZ4_dict);
LZ4_prepareTable(dict, 0, tableType);
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index d05760b..111a09b 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -336,7 +336,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
U32 const mlAdd = (matchLength>=19) ? ((matchLength-19) / 255) + 1 : 0;
U32 const cost = 1 + llAdd + ll + 2 + mlAdd;
if (start==NULL) start = *anchor; /* only works for single segment */
- //g_debuglog_enable = (pos >= 2228) & (pos <= 2262);
+ /* g_debuglog_enable = (pos >= 2228) & (pos <= 2262); */
DEBUGLOG(6, "pos:%7u -- literals:%3u, match:%4i, offset:%5u, cost:%3u + %u",
pos,
(U32)(*ip - *anchor), matchLength, (U32)(*ip-match),
@@ -1137,15 +1137,14 @@ static int LZ4HC_compress_optimal (
encode: /* cur, last_match_pos, best_mlen, best_off must be set */
assert(cur < LZ4_OPT_NUM);
assert(last_match_pos >= 1); /* == 1 when only one candidate */
- DEBUGLOG(6, "reverse traversal, looking for shortest path")
- DEBUGLOG(6, "last_match_pos = %i", last_match_pos);
+ DEBUGLOG(6, "reverse traversal, looking for shortest path (last_match_pos=%i)", last_match_pos);
{ int candidate_pos = cur;
int selected_matchLength = best_mlen;
int selected_offset = best_off;
while (1) { /* from end to beginning */
int const next_matchLength = opt[candidate_pos].mlen; /* can be 1, means literal */
int const next_offset = opt[candidate_pos].off;
- DEBUGLOG(6, "pos %i: sequence length %i", candidate_pos, selected_matchLength);
+ DEBUGLOG(7, "pos %i: sequence length %i", candidate_pos, selected_matchLength);
opt[candidate_pos].mlen = selected_matchLength;
opt[candidate_pos].off = selected_offset;
selected_matchLength = next_matchLength;
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 2e3ee92..a650277 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -632,13 +632,15 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary loaded with LZ4_loadDict()");
+ DISPLAYLEVEL(5, " compress %i bytes from buffer(%p) into dst(%p) using dict(%p) of size %i \n", blockSize, block, decodedBuffer, dict, dictSize);
LZ4_loadDict(&LZ4dict, dict, dictSize);
ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue should work : enough size available within output buffer");
/* Decompress with dictionary as external */
- FUZ_DISPLAYTEST("test LZ4_decompress_fast_usingDict(): decoding %i bytes, dict(%p) of size %i", blockSize, dict, dictSize);
+ FUZ_DISPLAYTEST("test LZ4_decompress_fast_usingDict() with dictionary as extDict");
+ DISPLAYLEVEL(5, " decoding %i bytes from buffer(%p) using dict(%p) of size %i \n", blockSize, decodedBuffer, dict, dictSize);
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
--
cgit v0.12
From db9aa785c51bbcae50c777e89fb537393bfca856 Mon Sep 17 00:00:00 2001
From: test4973
Date: Thu, 12 Apr 2018 16:12:21 -0700
Subject: fixed : counting matches which overlap extDict and prefix
---
lib/lz4.c | 27 +++++++++++++++++----------
1 file changed, 17 insertions(+), 10 deletions(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index e4a68cd..21892d3 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -601,11 +601,9 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
* than compressing without a gap. However, compressing with
* currentOffset == 0 is faster still, so we preserve that case.
*/
- DEBUGLOG(2, "tableType=%u, currentOffset=%u", cctx->tableType, cctx->currentOffset);
if (cctx->currentOffset != 0 && tableType == byU32) {
cctx->currentOffset += 64 KB;
}
- DEBUGLOG(2, "currentOffset: %u", cctx->currentOffset);
/* Finally, clear history */
cctx->dictCtx = NULL;
@@ -652,7 +650,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
const BYTE* dictBase = dictDirective == usingDictCtx ?
dictionary + dictSize - dictCtx->currentOffset : /* is it possible that dictCtx->currentOffset != dictCtx->dictSize ? */
dictionary + dictSize - startIndex;
- const BYTE* dictLowLimit;
BYTE* op = (BYTE*) dest;
BYTE* const olimit = op + maxOutputSize;
@@ -665,7 +662,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */
lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0);
- dictLowLimit = dictionary ? dictionary : lowLimit;
if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */
@@ -735,15 +731,16 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
/* there was no match, try the dictionary */
matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32);
match = dictBase + matchIndex;
- lowLimit = dictLowLimit;
+ lowLimit = dictionary;
} else {
match = base + matchIndex;
lowLimit = (const BYTE*)source;
}
} else if (dictDirective==usingExtDict) {
if (matchIndex < startIndex) {
+ DEBUGLOG(7, "extDict candidate: matchIndex=%5u < startIndex=%5u", matchIndex, startIndex);
match = dictBase + matchIndex;
- lowLimit = dictLowLimit;
+ lowLimit = dictionary;
} else {
match = base + matchIndex;
lowLimit = (const BYTE*)source;
@@ -786,6 +783,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
/* Copy Literals */
LZ4_wildCopy(op, anchor, op+litLength);
op+=litLength;
+ DEBUGLOG(6, "seq.start:%zi, literals=%u, match.start:%zi", anchor-(const BYTE*)source, litLength, ip-(const BYTE*)source);
}
_next_match:
@@ -793,29 +791,33 @@ _next_match:
if (maybe_ext_memSegment) { /* static test */
assert(offset <= MAX_DISTANCE && offset > 0);
LZ4_writeLE16(op, (U16)offset); op+=2;
+ DEBUGLOG(6, " with offset=%u (ext if > %zi)", offset, ip - (const BYTE*)source);
} else {
assert(ip-match <= MAX_DISTANCE);
LZ4_writeLE16(op, (U16)(ip - match)); op+=2;
+ DEBUGLOG(6, " with offset=%u (same segment)", (U32)(ip - match));
}
/* Encode MatchLength */
{ unsigned matchCode;
if ( (dictDirective==usingExtDict || dictDirective==usingDictCtx)
- && (lowLimit==dictionary) ) {
- const BYTE* limit;
- limit = ip + (dictEnd-match);
+ && (lowLimit==dictionary) /* match within extDict */ ) {
+ const BYTE* limit = ip + (dictEnd-match);
+ assert(dictEnd > match);
if (limit > matchlimit) limit = matchlimit;
matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit);
ip += MINMATCH + matchCode;
if (ip==limit) {
- unsigned const more = LZ4_count(ip, (const BYTE*)source, matchlimit);
+ unsigned const more = LZ4_count(limit, (const BYTE*)source, matchlimit);
matchCode += more;
ip += more;
}
+ DEBUGLOG(6, " with matchLength=%u starting in extDict", matchCode+MINMATCH);
} else {
matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit);
ip += MINMATCH + matchCode;
+ DEBUGLOG(6, " with matchLength=%u", matchCode+MINMATCH);
}
if ( outputLimited && /* Check output buffer overflow */
@@ -865,14 +867,18 @@ _next_match:
/* there was no match, try the dictionary */
matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32);
match = dictBase + matchIndex;
+ lowLimit = dictionary; /* required for match length counter */
} else {
match = base + matchIndex;
+ lowLimit = (const BYTE*)source; /* required for match length counter */
}
} else if (dictDirective==usingExtDict) {
if (matchIndex < startIndex) {
match = dictBase + matchIndex;
+ lowLimit = dictionary; /* required for match length counter */
} else {
match = base + matchIndex;
+ lowLimit = (const BYTE*)source; /* required for match length counter */
}
} else { /* single memory segment */
match = base + matchIndex;
@@ -885,6 +891,7 @@ _next_match:
*token=0;
if (maybe_ext_memSegment)
offset = current - matchIndex;
+ DEBUGLOG(6, "seq.start:%zi, literals=%u, match.start:%zi", anchor-(const BYTE*)source, 0, ip-(const BYTE*)source);
goto _next_match;
}
}
--
cgit v0.12
From 98811d606808da9f59affd990670ba34e5a9bee2 Mon Sep 17 00:00:00 2001
From: Cyan4973
Date: Fri, 13 Apr 2018 00:59:27 -0700
Subject: added sudo rights for low-mem-address tests
---
.travis.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.travis.yml b/.travis.yml
index a446420..0a876f9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,7 +14,7 @@ matrix:
env: Ubu=12.04cont Cmd='make -C tests test-lz4 test-lz4c test-fullbench' COMPILER=cc
- os: linux
- sudo: false
+ sudo: required
env: Ubu=12.04cont Cmd='sudo sysctl -w vm.mmap_min_addr="4096" && make -C tests test-frametest test-fuzzer' COMPILER=cc
- os: linux
--
cgit v0.12
From 57afa36795f478d0f9b069ad19b578761e3fb16a Mon Sep 17 00:00:00 2001
From: Cyan4973
Date: Fri, 13 Apr 2018 01:01:54 -0700
Subject: compatibility with gcc-4.4 string.h version
Someone found it would be a great idea to define there a global variable under the very generic name "index".
Cause problem with shadow warnings, so no variable can be named "index" now ...
Also : automatically update API manual
---
doc/lz4_manual.html | 94 ++++++++++++++++++++++++++++++++++++++++-------------
lib/lz4.c | 6 ++--
2 files changed, 75 insertions(+), 25 deletions(-)
diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index d14467f..f8639fe 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -15,8 +15,9 @@
Advanced Functions
Streaming Compression Functions
Streaming Decompression Functions
-Private definitions
-Obsolete Functions
+Unstable declarations
+Private definitions
+Obsolete Functions
Introduction
@@ -245,22 +246,80 @@ int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize,
-Private definitions
+Unstable declarations
+ Declarations in this section should be considered unstable.
+ Use at your own peril, etc., etc.
+ They may be removed in the future.
+ Their signatures may change.
+
+
+void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
+ When an LZ4_stream_t is known to be in a internally coherent state,
+ it can often be prepared for a new compression with almost no work, only
+ sometimes falling back to the full, expensive reset that is always required
+ when the stream is in an indeterminate state (i.e., the reset performed by
+ LZ4_resetStream()).
+
+ LZ4_streams are guaranteed to be in a valid state when:
+ - returned from LZ4_createStream()
+ - reset by LZ4_resetStream()
+ - memset(stream, 0, sizeof(LZ4_stream_t))
+ - the stream was in a valid state and was reset by LZ4_resetStream_fast()
+ - the stream was in a valid state and was then used in any compression call
+ that returned success
+ - the stream was in an indeterminate state and was used in a compression
+ call that fully reset the state (LZ4_compress_fast_extState()) and that
+ returned success
+
+
+
+int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
+ A variant of LZ4_compress_fast_extState().
+
+ Using this variant avoids an expensive initialization step. It is only safe
+ to call if the state buffer is known to be correctly initialized already
+ (see above comment on LZ4_resetStream_fast() for a definition of "correctly
+ initialized"). From a high level, the difference is that this function
+ initializes the provided state with a call to LZ4_resetStream_fast() while
+ LZ4_compress_fast_extState() starts with a call to LZ4_resetStream().
+
+
+
+void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream);
+ This is an experimental API that allows for the efficient use of a
+ static dictionary many times.
+
+ Rather than re-loading the dictionary buffer into a working context before
+ each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a
+ working LZ4_stream_t, this function introduces a no-copy setup mechanism,
+ in which the working stream references the dictionary stream in-place.
+
+ Several assumptions are made about the state of the dictionary stream.
+ Currently, only streams which have been prepared by LZ4_loadDict() should
+ be expected to work.
+
+ Alternatively, the provided dictionary stream pointer may be NULL, in which
+ case any existing dictionary stream is unset.
+
+ If a dictionary is provided, it replaces any pre-existing stream history.
+ The dictionary contents are the only history that can be referenced and
+ logically immediately precede the data compressed in the first subsequent
+ compression call.
+
+ The dictionary will only remain attached to the working stream through the
+ first compression call, at the end of which it is cleared. The dictionary
+ stream (and source buffer) must remain in-place / accessible / unchanged
+ through the completion of the first compression call on the stream.
+
+
+
+Private definitions
Do not use these definitions.
They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`.
Using these definitions will expose code to API and/or ABI break in future versions of the library.
typedef struct {
- uint32_t hashTable[LZ4_HASH_SIZE_U32];
- uint32_t currentOffset;
- uint32_t initCheck;
- const uint8_t* dictionary;
- uint8_t* bufferStart; /* obsolete, used for slideInputBuffer */
- uint32_t dictSize;
-} LZ4_stream_t_internal;
-
-typedef struct {
const uint8_t* externalDict;
size_t extDictSize;
const uint8_t* prefixEnd;
@@ -268,15 +327,6 @@ int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize,
} LZ4_streamDecode_t_internal;
typedef struct {
- unsigned int hashTable[LZ4_HASH_SIZE_U32];
- unsigned int currentOffset;
- unsigned int initCheck;
- const unsigned char* dictionary;
- unsigned char* bufferStart; /* obsolete, used for slideInputBuffer */
- unsigned int dictSize;
-} LZ4_stream_t_internal;
-
-typedef struct {
const unsigned char* externalDict;
size_t extDictSize;
const unsigned char* prefixEnd;
@@ -311,7 +361,7 @@ union LZ4_streamDecode_u {
-Obsolete Functions
+Obsolete Functions
#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS
# define LZ4_DEPRECATED(message) /* disable deprecation warnings */
diff --git a/lib/lz4.c b/lib/lz4.c
index 21892d3..f55e4e1 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -517,15 +517,15 @@ LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tab
return LZ4_hash4(LZ4_read32(p), tableType);
}
-static void LZ4_putIndexOnHash(U32 index, U32 h, void* tableBase, tableType_t const tableType)
+static void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t const tableType)
{
switch (tableType)
{
default: /* fallthrough */
case clearedTable: /* fallthrough */
case byPtr: { /* illegal! */ assert(0); return; }
- case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = index; return; }
- case byU16: { U16* hashTable = (U16*) tableBase; assert(index < 65536); hashTable[h] = (U16)index; return; }
+ case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = idx; return; }
+ case byU16: { U16* hashTable = (U16*) tableBase; assert(idx < 65536); hashTable[h] = (U16)idx; return; }
}
}
--
cgit v0.12
From 54ec83ce1f62014398b76a441d1ff1212dad9604 Mon Sep 17 00:00:00 2001
From: Yann Collet
Date: Fri, 13 Apr 2018 02:10:53 -0700
Subject: fixed potential ptrdiff_t overflow (32-bits mode)
Also removed pointer comparison, which should solve #485
---
lib/lz4.c | 25 +++++++++++--------------
1 file changed, 11 insertions(+), 14 deletions(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index f55e4e1..c48aca4 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1315,15 +1315,14 @@ void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dic
}
-static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src)
+static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, int nextSize)
{
- if ((LZ4_dict->currentOffset > 0x80000000) ||
- ((uptrval)LZ4_dict->currentOffset > (uptrval)src)) { /* address space overflow */
+ if (LZ4_dict->currentOffset + nextSize > 0x80000000) { /* potential ptrdiff_t overflow (32-bits mode) */
/* rescale hash table */
U32 const delta = LZ4_dict->currentOffset - 64 KB;
const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize;
int i;
- DEBUGLOG(4, "LZ4_renormDictT %p", LZ4_dict);
+ DEBUGLOG(4, "LZ4_renormDictT");
for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0;
else LZ4_dict->hashTable[i] -= delta;
@@ -1341,10 +1340,8 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse;
const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
- const BYTE* smallest = (const BYTE*) source;
if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */
- if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd;
- LZ4_renormDictT(streamPtr, smallest);
+ LZ4_renormDictT(streamPtr, inputSize); /* avoid index overflow */
if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
/* Check overlapping input/dictionary space */
@@ -1377,7 +1374,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
if (inputSize > 4 KB) {
/* For compressing large blobs, it is faster to pay the setup
* cost to copy the dictionary's tables into the active context,
- * so that the compression loop is only looking in one table.
+ * so that the compression loop is only looking into one table.
*/
memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
@@ -1398,8 +1395,8 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
}
-/* Hidden debug function, to force external dictionary mode */
-int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize)
+/* Hidden debug function, to force-test external dictionary mode */
+int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize)
{
LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse;
int result;
@@ -1407,16 +1404,16 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
const BYTE* smallest = dictEnd;
if (smallest > (const BYTE*) source) smallest = (const BYTE*) source;
- LZ4_renormDictT(streamPtr, smallest);
+ LZ4_renormDictT(streamPtr, srcSize);
if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
- result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, dictSmall, 1);
+ result = LZ4_compress_generic(streamPtr, source, dest, srcSize, 0, notLimited, byU32, usingExtDict, dictSmall, 1);
} else {
- result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
+ result = LZ4_compress_generic(streamPtr, source, dest, srcSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
}
streamPtr->dictionary = (const BYTE*)source;
- streamPtr->dictSize = (U32)inputSize;
+ streamPtr->dictSize = (U32)srcSize;
return result;
}
--
cgit v0.12
From c40bac31d3886c2f69d0364823b6d6aaa972ee5b Mon Sep 17 00:00:00 2001
From: Yann Collet
Date: Fri, 13 Apr 2018 02:26:14 -0700
Subject: added comment on variables required after _next_match
---
lib/lz4.c | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/lib/lz4.c b/lib/lz4.c
index c48aca4..54d037c 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -787,6 +787,14 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
}
_next_match:
+ /* at this stage, the following variables must be correctly set :
+ * - ip : at start of LZ operation
+ * - match : at start of previous pattern occurence; can be within current prefix, or within extDict
+ * - offset : if maybe_ext_memSegment==1 (constant)
+ * - lowLimit : must be == dictionary to mean "match is within extDict"; must be == source otherwise
+ * - token and *token : position to write 4-bits for match length; higher 4-bits for literal length supposed already written
+ */
+
/* Encode Offset */
if (maybe_ext_memSegment) { /* static test */
assert(offset <= MAX_DISTANCE && offset > 0);
--
cgit v0.12
From d2bcfa31f525aaa11c2d248af0ba487791399c1f Mon Sep 17 00:00:00 2001
From: Yann Collet
Date: Fri, 13 Apr 2018 02:45:32 -0700
Subject: fixed minor unused variable warning
---
lib/lz4.c | 3 ---
1 file changed, 3 deletions(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index 54d037c..e8f9831 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1408,10 +1408,7 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
{
LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse;
int result;
- const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
- const BYTE* smallest = dictEnd;
- if (smallest > (const BYTE*) source) smallest = (const BYTE*) source;
LZ4_renormDictT(streamPtr, srcSize);
if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
--
cgit v0.12
From 6dd64e0776dccf5c19e8438625657abf0e6f3048 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte"
Date: Thu, 12 Apr 2018 18:23:01 -0400
Subject: Add Tests for LZ4_attach_dictionary and Friends
---
tests/fuzzer.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 78 insertions(+)
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index c134fe3..98b3d9f 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -43,6 +43,7 @@
#include /* fgets, sscanf */
#include /* strcmp */
#include /* clock_t, clock, CLOCKS_PER_SEC */
+#define LZ4_STATIC_LINKING_ONLY
#define LZ4_HC_STATIC_LINKING_ONLY
#include "lz4hc.h"
#define XXH_STATIC_LINKING_ONLY
@@ -402,6 +403,11 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed");
+ /* Test compression using fast reset external state*/
+ FUZ_DISPLAYTEST;
+ ret = LZ4_compress_fast_extState_fastReset(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
+ FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState_fastReset() failed");
+
/* Test compression */
FUZ_DISPLAYTEST;
ret = LZ4_compress_default(block, compressedBuffer, blockSize, (int)compressedBufferSize);
@@ -626,6 +632,78 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize);
} }
+ /* Compress using external dictionary stream */
+ FUZ_DISPLAYTEST;
+ {
+ LZ4_stream_t LZ4_stream;
+
+ LZ4_loadDict(&LZ4dict, dict, dictSize);
+
+ LZ4_resetStream(&LZ4_stream);
+ LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
+ blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
+ FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue using extDictCtx failed");
+
+ FUZ_DISPLAYTEST;
+ LZ4_resetStream(&LZ4_stream);
+ LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
+ ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
+ FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using extDictCtx should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
+
+ FUZ_DISPLAYTEST;
+ LZ4_resetStream(&LZ4_stream);
+ LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
+ ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
+ FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
+ FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx should work : enough size available within output buffer");
+
+ FUZ_DISPLAYTEST;
+ LZ4_resetStream_fast(&LZ4_stream);
+ LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
+ ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
+ FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
+ FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx with re-used context should work : enough size available within output buffer");
+ }
+
+ /* Decompress with dictionary as external */
+ FUZ_DISPLAYTEST;
+ decodedBuffer[blockSize] = 0;
+ ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
+ FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
+ FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
+ crcCheck = XXH32(decodedBuffer, blockSize, 0);
+ if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
+ FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
+
+ FUZ_DISPLAYTEST;
+ decodedBuffer[blockSize] = 0;
+ ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
+ FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
+ FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
+ crcCheck = XXH32(decodedBuffer, blockSize, 0);
+ FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+
+ FUZ_DISPLAYTEST;
+ decodedBuffer[blockSize-1] = 0;
+ ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
+ FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
+ FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
+
+ FUZ_DISPLAYTEST;
+ decodedBuffer[blockSize-1] = 0;
+ ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
+ FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
+ FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
+
+ FUZ_DISPLAYTEST;
+ { U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
+ if ((U32)blockSize > missingBytes) {
+ decodedBuffer[blockSize-missingBytes] = 0;
+ ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-missingBytes, dict, dictSize);
+ FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%u byte)", missingBytes);
+ FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize);
+ } }
+
/* Compress HC using External dictionary */
FUZ_DISPLAYTEST;
dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */
--
cgit v0.12
From 9f0f6b89bb27d4d6c5d12a7142fc330f3d1b3a10 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte"
Date: Thu, 12 Apr 2018 19:17:53 -0400
Subject: Further Test that ExtDictCtx Mode Produces the Exact Same Output
---
tests/fuzzer.c | 23 +++++++++++++++++++++--
1 file changed, 21 insertions(+), 2 deletions(-)
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 98b3d9f..db05738 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -303,9 +303,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
U32 testNb = 0;
U32 randState = FUZ_rand(&coreRandState) ^ PRIME3;
int const blockSize = (FUZ_rand(&randState) % (FUZ_MAX_BLOCK_SIZE-1)) + 1;
- int const blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize);
+ int const blockStart = (FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize - 1)) + 1;
int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
- int const dictSize = MIN(dictSizeRand, blockStart);
+ int const dictSize = MIN(dictSizeRand, blockStart - 1);
int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1);
char* const block = ((char*)CNBuffer) + blockStart;
const char* dict = block - dictSize;
@@ -636,6 +636,13 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_DISPLAYTEST;
{
LZ4_stream_t LZ4_stream;
+ int expectedSize;
+ U32 expectedCrc;
+
+ LZ4_loadDict(&LZ4dict, dict, dictSize);
+ expectedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
+ FUZ_CHECKTEST(expectedSize<=0, "LZ4_compress_fast_continue reference compression for extDictCtx should have succeeded");
+ expectedCrc = XXH32(compressedBuffer, expectedSize, 0);
LZ4_loadDict(&LZ4dict, dict, dictSize);
@@ -644,6 +651,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue using extDictCtx failed");
+ /* In the future, it might be desirable to let extDictCtx mode's
+ * output diverge from the output generated by regular extDict mode.
+ * Until that time, this comparison serves as a good regression
+ * test.
+ */
+ FUZ_CHECKTEST(blockContinueCompressedSize != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output (%d expected vs %d actual)", expectedSize, blockContinueCompressedSize);
+ FUZ_CHECKTEST(XXH32(compressedBuffer, blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
+
FUZ_DISPLAYTEST;
LZ4_resetStream(&LZ4_stream);
LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
@@ -656,6 +671,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx should work : enough size available within output buffer");
+ FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output");
+ FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
FUZ_DISPLAYTEST;
LZ4_resetStream_fast(&LZ4_stream);
@@ -663,6 +680,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx with re-used context should work : enough size available within output buffer");
+ FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output");
+ FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
}
/* Decompress with dictionary as external */
--
cgit v0.12
From e9280647976ca468ab67ffbf3dd9c8e533fe23bf Mon Sep 17 00:00:00 2001
From: Yann Collet
Date: Mon, 16 Apr 2018 15:11:28 -0700
Subject: fixed gcc performance regression
---
lib/lz4.c | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index e8f9831..c2012f6 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -660,6 +660,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
/* Init conditions */
if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */
if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */
+ assert(acceleration >= 1);
lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0);
@@ -712,6 +713,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
} else { /* byU32, byU16 */
const BYTE* forwardIp = ip;
+ unsigned step = 1;
unsigned searchMatchNb = acceleration << LZ4_skipTrigger;
do {
U32 const h = forwardH;
@@ -720,8 +722,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
assert(matchIndex <= current);
assert(forwardIp - base < (ptrdiff_t)(2 GB - 1));
ip = forwardIp;
- assert(searchMatchNb >= (1<> LZ4_skipTrigger);
+ forwardIp += step;
+ step = (searchMatchNb++ >> LZ4_skipTrigger);
if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals;
assert(ip < mflimitPlusOne);
--
cgit v0.12
From 4aff9b10b56e44b2ff6326f611c3659565812d80 Mon Sep 17 00:00:00 2001
From: Yann Collet
Date: Mon, 16 Apr 2018 16:14:28 -0700
Subject: fixed fuzzer tests
which were modified in parallel within branc `dev`
---
tests/fuzzer.c | 32 +++++++++++++++++---------------
1 file changed, 17 insertions(+), 15 deletions(-)
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 83ab50b..7721345 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -454,7 +454,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed");
/* Test compression using fast reset external state*/
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
ret = LZ4_compress_fast_extState_fastReset(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState_fastReset() failed");
@@ -687,7 +687,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
} }
/* Compress using external dictionary stream */
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
{
LZ4_stream_t LZ4_stream;
int expectedSize;
@@ -713,13 +713,13 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(blockContinueCompressedSize != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output (%d expected vs %d actual)", expectedSize, blockContinueCompressedSize);
FUZ_CHECKTEST(XXH32(compressedBuffer, blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
LZ4_resetStream(&LZ4_stream);
LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using extDictCtx should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
LZ4_resetStream(&LZ4_stream);
LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
@@ -728,7 +728,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output");
FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
LZ4_resetStream_fast(&LZ4_stream);
LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
@@ -739,36 +739,38 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
}
/* Decompress with dictionary as external */
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
- crcCheck = XXH32(decodedBuffer, blockSize, 0);
- if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
- FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
+ { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+ if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
+ FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
+ }
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
- crcCheck = XXH32(decodedBuffer, blockSize, 0);
- FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+ { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+ FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+ }
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
- FUZ_DISPLAYTEST;
+ FUZ_DISPLAYTEST();
{ U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
if ((U32)blockSize > missingBytes) {
decodedBuffer[blockSize-missingBytes] = 0;
--
cgit v0.12
From a3aeb34184e20d51616beccfcbbe7aade3cc3a64 Mon Sep 17 00:00:00 2001
From: Yann Collet
Date: Mon, 16 Apr 2018 16:54:03 -0700
Subject: fixed minor format warnings
---
lib/lz4.c | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index c2012f6..980a5fd 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -785,7 +785,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
/* Copy Literals */
LZ4_wildCopy(op, anchor, op+litLength);
op+=litLength;
- DEBUGLOG(6, "seq.start:%zi, literals=%u, match.start:%zi", anchor-(const BYTE*)source, litLength, ip-(const BYTE*)source);
+ DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", (int)(anchor-(const BYTE*)source), litLength, (int)(ip-(const BYTE*)source));
}
_next_match:
@@ -801,7 +801,7 @@ _next_match:
if (maybe_ext_memSegment) { /* static test */
assert(offset <= MAX_DISTANCE && offset > 0);
LZ4_writeLE16(op, (U16)offset); op+=2;
- DEBUGLOG(6, " with offset=%u (ext if > %zi)", offset, ip - (const BYTE*)source);
+ DEBUGLOG(6, " with offset=%u (ext if > %i)", offset, (int)(ip - (const BYTE*)source));
} else {
assert(ip-match <= MAX_DISTANCE);
LZ4_writeLE16(op, (U16)(ip - match)); op+=2;
@@ -901,7 +901,7 @@ _next_match:
*token=0;
if (maybe_ext_memSegment)
offset = current - matchIndex;
- DEBUGLOG(6, "seq.start:%zi, literals=%u, match.start:%zi", anchor-(const BYTE*)source, 0, ip-(const BYTE*)source);
+ DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source));
goto _next_match;
}
}
--
cgit v0.12
From 444211d2599a2be59e3f50418b46ec2431288e9a Mon Sep 17 00:00:00 2001
From: Yann Collet
Date: Mon, 16 Apr 2018 17:15:02 -0700
Subject: edited a few traces for debugging
---
lib/lz4.c | 14 +++++++-------
tests/fuzzer.c | 6 +++---
2 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index 980a5fd..dca4d69 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -588,12 +588,12 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
|| tableType == byPtr
|| inputSize >= 4 KB)
{
- DEBUGLOG(4, "Resetting table in %p", cctx);
+ DEBUGLOG(4, "LZ4_prepareTable: Resetting table in %p", cctx);
MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
cctx->currentOffset = 0;
cctx->tableType = clearedTable;
} else {
- DEBUGLOG(4, "Re-use hash table (no reset)");
+ DEBUGLOG(4, "LZ4_prepareTable: Re-use hash table (no reset)");
}
}
@@ -799,13 +799,13 @@ _next_match:
/* Encode Offset */
if (maybe_ext_memSegment) { /* static test */
+ DEBUGLOG(6, " with offset=%u (ext if > %i)", offset, (int)(ip - (const BYTE*)source));
assert(offset <= MAX_DISTANCE && offset > 0);
LZ4_writeLE16(op, (U16)offset); op+=2;
- DEBUGLOG(6, " with offset=%u (ext if > %i)", offset, (int)(ip - (const BYTE*)source));
} else {
+ DEBUGLOG(6, " with offset=%u (same segment)", (U32)(ip - match));
assert(ip-match <= MAX_DISTANCE);
LZ4_writeLE16(op, (U16)(ip - match)); op+=2;
- DEBUGLOG(6, " with offset=%u (same segment)", (U32)(ip - match));
}
/* Encode MatchLength */
@@ -823,11 +823,11 @@ _next_match:
matchCode += more;
ip += more;
}
- DEBUGLOG(6, " with matchLength=%u starting in extDict", matchCode+MINMATCH);
+ DEBUGLOG(6, " with matchLength=%u starting in extDict", matchCode+MINMATCH);
} else {
matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit);
ip += MINMATCH + matchCode;
- DEBUGLOG(6, " with matchLength=%u", matchCode+MINMATCH);
+ DEBUGLOG(6, " with matchLength=%u", matchCode+MINMATCH);
}
if ( outputLimited && /* Check output buffer overflow */
@@ -1259,7 +1259,7 @@ LZ4_stream_t* LZ4_createStream(void)
void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
{
- DEBUGLOG(5, "LZ4_resetStream %p", LZ4_stream);
+ DEBUGLOG(5, "LZ4_resetStream (ctx:%p)", LZ4_stream);
MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
}
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 7721345..244cc4f 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -687,19 +687,19 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
} }
/* Compress using external dictionary stream */
- FUZ_DISPLAYTEST();
{
LZ4_stream_t LZ4_stream;
int expectedSize;
U32 expectedCrc;
+ FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_loadDict()");
LZ4_loadDict(&LZ4dict, dict, dictSize);
expectedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
FUZ_CHECKTEST(expectedSize<=0, "LZ4_compress_fast_continue reference compression for extDictCtx should have succeeded");
expectedCrc = XXH32(compressedBuffer, expectedSize, 0);
+ FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary()");
LZ4_loadDict(&LZ4dict, dict, dictSize);
-
LZ4_resetStream(&LZ4_stream);
LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
@@ -713,7 +713,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(blockContinueCompressedSize != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output (%d expected vs %d actual)", expectedSize, blockContinueCompressedSize);
FUZ_CHECKTEST(XXH32(compressedBuffer, blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
- FUZ_DISPLAYTEST();
+ FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary(), but output buffer is 1 byte too short");
LZ4_resetStream(&LZ4_stream);
LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
--
cgit v0.12
From da3b5ba6f0014564b7312511e441067ba9429733 Mon Sep 17 00:00:00 2001
From: Yann Collet
Date: Mon, 16 Apr 2018 23:59:42 -0700
Subject: fixed dictCtx compression
---
lib/lz4.c | 19 ++++++++++++-------
1 file changed, 12 insertions(+), 7 deletions(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index dca4d69..0590de4 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -602,6 +602,7 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
* currentOffset == 0 is faster still, so we preserve that case.
*/
if (cctx->currentOffset != 0 && tableType == byU32) {
+ DEBUGLOG(5, "LZ4_prepareTable: adding 64KB to currentOffset");
cctx->currentOffset += 64 KB;
}
@@ -636,8 +637,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
dictDirective == usingDictCtx ? dictCtx->dictionary : cctx->dictionary;
const U32 dictSize =
dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize;
+ const U32 dictDelta = usingDictCtx ? startIndex - dictCtx->currentOffset : 0; /* make indexes in dictCtx comparable with index in current context */
- int const maybe_ext_memSegment = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx);
+ int const maybe_extMem = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx);
U32 const prefixIdxLimit = startIndex - dictSize; /* used when dictDirective == dictSmall */
const BYTE* const dictEnd = dictionary + dictSize;
const BYTE* anchor = (const BYTE*) source;
@@ -648,7 +650,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
/* the dictCtx currentOffset is indexed on the start of the dictionary,
* while a dictionary in the current context precedes the currentOffset */
const BYTE* dictBase = dictDirective == usingDictCtx ?
- dictionary + dictSize - dictCtx->currentOffset : /* is it possible that dictCtx->currentOffset != dictCtx->dictSize ? */
+ dictionary + dictSize - dictCtx->currentOffset : /* is it possible that dictCtx->currentOffset != dictCtx->dictSize ? Yes if the dictionary context is not reset */
dictionary + dictSize - startIndex;
BYTE* op = (BYTE*) dest;
@@ -657,6 +659,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
U32 offset = 0;
U32 forwardH;
+ DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, tableType=%u", inputSize, tableType);
/* Init conditions */
if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */
if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */
@@ -731,8 +734,10 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
if (dictDirective == usingDictCtx) {
if (matchIndex < startIndex) {
/* there was no match, try the dictionary */
+ assert(tableType == byU32);
matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32);
match = dictBase + matchIndex;
+ matchIndex += dictDelta; /* make dictCtx index comparable with current context */
lowLimit = dictionary;
} else {
match = base + matchIndex;
@@ -758,7 +763,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE); /* too_far presumed impossible with byU16 */
if (LZ4_read32(match) == LZ4_read32(ip)) {
- if (maybe_ext_memSegment) offset = current - matchIndex;
+ if (maybe_extMem) offset = current - matchIndex;
break; /* match found */
}
@@ -798,7 +803,7 @@ _next_match:
*/
/* Encode Offset */
- if (maybe_ext_memSegment) { /* static test */
+ if (maybe_extMem) { /* static test */
DEBUGLOG(6, " with offset=%u (ext if > %i)", offset, (int)(ip - (const BYTE*)source));
assert(offset <= MAX_DISTANCE && offset > 0);
LZ4_writeLE16(op, (U16)offset); op+=2;
@@ -878,6 +883,7 @@ _next_match:
matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32);
match = dictBase + matchIndex;
lowLimit = dictionary; /* required for match length counter */
+ matchIndex += dictDelta;
} else {
match = base + matchIndex;
lowLimit = (const BYTE*)source; /* required for match length counter */
@@ -899,8 +905,7 @@ _next_match:
&& (LZ4_read32(match) == LZ4_read32(ip)) ) {
token=op++;
*token=0;
- if (maybe_ext_memSegment)
- offset = current - matchIndex;
+ if (maybe_extMem) offset = current - matchIndex;
DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source));
goto _next_match;
}
@@ -1285,7 +1290,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
const BYTE* const dictEnd = p + dictSize;
const BYTE* base;
- DEBUGLOG(4, "LZ4_loadDict (%p into %p)", dictionary, LZ4_dict);
+ DEBUGLOG(4, "LZ4_loadDict (%i bytes from %p into %p)", dictSize, dictionary, LZ4_dict);
LZ4_prepareTable(dict, 0, tableType);
--
cgit v0.12
From aedc44780468c11d90114bc1d6124af545bfc652 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte"
Date: Tue, 17 Apr 2018 14:01:44 -0400
Subject: Always Bump Offset by 64 KB in LZ4_loadDict()
This actually ensures the guarantee referred to in the comment in
LZ4_compress_fast_continue().
---
lib/lz4.c | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index 4b0efb1..0ce05da 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1173,11 +1173,18 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
LZ4_prepareTable(dict, 0, tableType);
+ /* We always increment the offset by 64 KB, since, if the dict is longer,
+ * we truncate it to the last 64k, and if it's shorter, we still want to
+ * advance by a whole window length so we can provide the guarantee that
+ * there are only valid offsets in the window, which allows an optimization
+ * in LZ4_compress_fast_continue() where it uses noDictIssue even when the
+ * dictionary isn't a full 64k. */
+
if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;
- base = p - dict->currentOffset;
+ base = dictEnd - 64 KB - dict->currentOffset;
dict->dictionary = p;
dict->dictSize = (U32)(dictEnd - p);
- dict->currentOffset += dict->dictSize;
+ dict->currentOffset += 64 KB;
dict->tableType = tableType;
if (dictSize < (int)HASH_UNIT) {
--
cgit v0.12
From 152064218361e5762fd67b5de425707fdc47095b Mon Sep 17 00:00:00 2001
From: Yann Collet
Date: Tue, 17 Apr 2018 15:29:17 -0700
Subject: fix matchIndex overflow
can happen with dictCtx
---
lib/lz4.c | 16 ++++------------
1 file changed, 4 insertions(+), 12 deletions(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index b426545..c799596 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -758,9 +758,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
forwardH = LZ4_hashPosition(forwardIp, tableType);
LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
- if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue; /* match outside of valid area */
- if ((tableType != byU16) && (matchIndex+MAX_DISTANCE < current)) continue; /* too far */
- if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE); /* too_far presumed impossible with byU16 */
+ if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue; /* match outside of valid area */
+ if ((tableType != byU16) && (current - matchIndex > MAX_DISTANCE)) continue; /* too far - note: works even if matchIndex overflows */
+ if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE); /* too_far presumed impossible with byU16 */
if (LZ4_read32(match) == LZ4_read32(ip)) {
if (maybe_extMem) offset = current - matchIndex;
@@ -861,7 +861,6 @@ _next_match:
/* Fill table */
LZ4_putPosition(ip-2, cctx->hashTable, tableType, base);
-#if 1
/* Test next position */
if (tableType == byPtr) {
@@ -901,7 +900,7 @@ _next_match:
}
LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1)
- && ((tableType==byU16) ? 1 : (matchIndex+MAX_DISTANCE >= current))
+ && ((tableType==byU16) ? 1 : (current - matchIndex <= MAX_DISTANCE))
&& (LZ4_read32(match) == LZ4_read32(ip)) ) {
token=op++;
*token=0;
@@ -914,13 +913,6 @@ _next_match:
/* Prepare next loop */
forwardH = LZ4_hashPosition(++ip, tableType);
-#else
-
- /* Prepare next loop */
- forwardH = LZ4_hashPosition(ip, tableType);
-
-#endif
-
}
_last_literals:
--
cgit v0.12
From 88cca1723e76c8f5031954ba07b28447c0cb55d8 Mon Sep 17 00:00:00 2001
From: Yann Collet
Date: Tue, 17 Apr 2018 16:18:37 -0700
Subject: fix dictDelta setting error
wrong test
---
lib/lz4.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index c799596..e7553ed 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -637,7 +637,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
dictDirective == usingDictCtx ? dictCtx->dictionary : cctx->dictionary;
const U32 dictSize =
dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize;
- const U32 dictDelta = usingDictCtx ? startIndex - dictCtx->currentOffset : 0; /* make indexes in dictCtx comparable with index in current context */
+ const U32 dictDelta = (dictDirective == usingDictCtx) ? startIndex - dictCtx->currentOffset : 0; /* make indexes in dictCtx comparable with index in current context */
int const maybe_extMem = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx);
U32 const prefixIdxLimit = startIndex - dictSize; /* used when dictDirective == dictSmall */
--
cgit v0.12
From 5ad4599c5ad18d2408a6ccc545c45a36b99f0c6f Mon Sep 17 00:00:00 2001
From: Yann Collet
Date: Tue, 17 Apr 2018 16:47:56 -0700
Subject: fixed LZ4_compress_fast_extState_fastReset() in 32-bit mode
---
lib/lz4.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index e7553ed..33aa5c7 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -983,7 +983,7 @@ int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst
return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
}
} else {
- const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+ const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
LZ4_prepareTable(ctx, srcSize, tableType);
return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
}
@@ -997,7 +997,7 @@ int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst
return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
}
} else {
- const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+ const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
LZ4_prepareTable(ctx, srcSize, tableType);
return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
}
--
cgit v0.12
From ea6ed46fc273a7b10870caeb515622737eb1c572 Mon Sep 17 00:00:00 2001
From: Dmitrii Rodionov
Date: Wed, 18 Apr 2018 12:20:56 +0300
Subject: Wrap likely/unlikely macroses with #ifndef
It prevent redefine error when project using lz4 has its own likely/unlikely
macroses.
---
lib/lz4.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/lib/lz4.c b/lib/lz4.c
index 0ce05da..1e96931 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -147,8 +147,12 @@
# define expect(expr,value) (expr)
#endif
+#ifndef likely
#define likely(expr) expect((expr) != 0, 1)
+#endif
+#ifndef unlikely
#define unlikely(expr) expect((expr) != 0, 0)
+#endif
/*-************************************
--
cgit v0.12
From 4785bd6a3508c6e10c5cee7126fd112a0ec44599 Mon Sep 17 00:00:00 2001
From: Yann Collet
Date: Wed, 18 Apr 2018 16:49:27 -0700
Subject: minor length reduction of several large lines
---
doc/lz4frame_manual.html | 50 ++++++++++++++++++-------------
lib/Makefile | 4 +++
lib/lz4.c | 76 +++++++++++++++++++++++++++++++++---------------
lib/lz4frame.h | 50 ++++++++++++++++++-------------
4 files changed, 115 insertions(+), 65 deletions(-)
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index f0f7f5a..459bac8 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -30,9 +30,9 @@
Error management
-unsigned LZ4F_isError(LZ4F_errorCode_t code); /**< tells if a `LZ4F_errorCode_t` function result is an error code */
+unsigned LZ4F_isError(LZ4F_errorCode_t code); /**< tells when a function result is an error code */
-const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return error code string; useful for debugging */
+const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return error code string; for debugging */
Frame compression types
@@ -74,13 +74,13 @@
} LZ4F_frameType_t;
typedef struct {
- LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
- LZ4F_blockMode_t blockMode; /* LZ4F_blockLinked, LZ4F_blockIndependent ; 0 == default */
- LZ4F_contentChecksum_t contentChecksumFlag; /* if enabled, frame is terminated with a 32-bits checksum of decompressed data ; 0 == disabled (default) */
- LZ4F_frameType_t frameType; /* read-only field : LZ4F_frame or LZ4F_skippableFrame */
- unsigned long long contentSize; /* Size of uncompressed content ; 0 == unknown */
- unsigned dictID; /* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */
- LZ4F_blockChecksum_t blockChecksumFlag; /* if enabled, each block is followed by a checksum of block's compressed data ; 0 == disabled (default) */
+ LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB; 0 == default */
+ LZ4F_blockMode_t blockMode; /* LZ4F_blockLinked, LZ4F_blockIndependent; 0 == default */
+ LZ4F_contentChecksum_t contentChecksumFlag; /* 1: frame terminated with 32-bit checksum of decompressed data; 0: disabled (default) */
+ LZ4F_frameType_t frameType; /* read-only field : LZ4F_frame or LZ4F_skippableFrame */
+ unsigned long long contentSize; /* Size of uncompressed content ; 0 == unknown */
+ unsigned dictID; /* Dictionary ID, sent by compressor to help decoder select correct dictionary; 0 == no dictID provided */
+ LZ4F_blockChecksum_t blockChecksumFlag; /* 1: each block followed by a checksum of block's compressed data; 0: disabled (default) */
} LZ4F_frameInfo_t;
makes it possible to set or read frame parameters.
It's not required to set all fields, as long as the structure was initially memset() to zero.
@@ -89,7 +89,7 @@
typedef struct {
LZ4F_frameInfo_t frameInfo;
- int compressionLevel; /* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */
+ int compressionLevel; /* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */
unsigned autoFlush; /* 1 == always flush, to reduce usage of internal buffers */
unsigned reserved[4]; /* must be zero for forward compatibility */
} LZ4F_preferences_t;
@@ -165,34 +165,42 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
-size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr);
+size_t LZ4F_compressUpdate(LZ4F_cctx* cctx,
+ void* dstBuffer, size_t dstCapacity,
+ const void* srcBuffer, size_t srcSize,
+ const LZ4F_compressOptions_t* cOptPtr);
LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
- An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations.
+ Important rule: dstCapacity MUST be large enough to ensure operation success even in worst case situations.
This value is provided by LZ4F_compressBound().
If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
- LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized.
+ LZ4F_compressUpdate() doesn't guarantee error recovery.
+ When an error occurs, compression context must be freed or resized.
`cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
@return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
or an error code if it fails (which can be tested using LZ4F_isError())
-size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
+size_t LZ4F_flush(LZ4F_cctx* cctx,
+ void* dstBuffer, size_t dstCapacity,
+ const LZ4F_compressOptions_t* cOptPtr);
When data must be generated and sent immediately, without waiting for a block to be completely filled,
it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.
`dstCapacity` must be large enough to ensure the operation will be successful.
`cOptPtr` is optional : it's possible to provide NULL, all options will be set to default.
- @return : number of bytes written into dstBuffer (it can be zero, which means there was no data stored within cctx)
+ @return : nb of bytes written into dstBuffer (can be zero, when there is no data stored within cctx)
or an error code if it fails (which can be tested using LZ4F_isError())
-size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
+size_t LZ4F_compressEnd(LZ4F_cctx* cctx,
+ void* dstBuffer, size_t dstCapacity,
+ const LZ4F_compressOptions_t* cOptPtr);
To properly finish an LZ4 frame, invoke LZ4F_compressEnd().
It will flush whatever data remained within `cctx` (like LZ4_flush())
and properly finalize the frame, with an endMark and a checksum.
`cOptPtr` is optional : NULL can be provided, in which case all options will be set to default.
- @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled)
+ @return : nb of bytes written into dstBuffer, necessarily >= 4 (endMark),
or an error code if it fails (which can be tested using LZ4F_isError())
A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.
@@ -201,7 +209,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
Decompression functions
typedef struct {
- unsigned stableDst; /* pledge that at least 64KB+64Bytes of previously decompressed data remain unmodifed where it was decoded. This optimization skips storage operations in tmp buffers */
+ unsigned stableDst; /* pledges that last 64KB decompressed data will remain available unmodified. This optimization skips storage operations in tmp buffers. */
unsigned reserved[3]; /* must be set to zero for forward compatibility */
} LZ4F_decompressOptions_t;
@@ -212,7 +220,7 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
The function provides a pointer to an allocated and initialized LZ4F_dctx object.
The result is an errorCode, which can be tested using LZ4F_isError().
dctx memory can be released using LZ4F_freeDecompressionContext();
- The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
+ Result of LZ4F_freeDecompressionContext() indicates current state of decompressionContext when being released.
That is, it should be == 0 if decompression has been completed fully and correctly.
@@ -252,8 +260,8 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
The function will read up to *srcSizePtr bytes from srcBuffer,
and decompress data into dstBuffer, of capacity *dstSizePtr.
- The number of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value).
- The number of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value).
+ The nb of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value).
+ The nb of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value).
The function does not necessarily read all input bytes, so always check value in *srcSizePtr.
Unconsumed source data must be presented again in subsequent invocations.
diff --git a/lib/Makefile b/lib/Makefile
index dd33f50..7b31239 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -116,6 +116,10 @@ clean:
#-----------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
+.PHONY: listL120
+listL120: # extract lines >= 120 characters in *.{c,h}, by Takayuki Matsuoka (note : $$, for Makefile compatibility)
+ find . -type f -name '*.c' -o -name '*.h' | while read -r filename; do awk 'length > 120 {print FILENAME "(" FNR "): " $$0}' $$filename; done
+
DESTDIR ?=
# directory variables : GNU conventions prefer lowercase
# see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
diff --git a/lib/lz4.c b/lib/lz4.c
index 33aa5c7..af3221c 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -69,7 +69,9 @@
* Prefer these methods in priority order (0 > 1 > 2)
*/
#ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally */
-# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
+# if defined(__GNUC__) && \
+ ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \
+ || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
# define LZ4_FORCE_MEMORY_ACCESS 2
# elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || defined(__GNUC__)
# define LZ4_FORCE_MEMORY_ACCESS 1
@@ -80,7 +82,7 @@
* LZ4_FORCE_SW_BITCOUNT
* Define this parameter if your target system or compiler does not support hardware bit count
*/
-#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */
+#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for WinCE doesn't support Hardware bit count */
# define LZ4_FORCE_SW_BITCOUNT
#endif
@@ -318,7 +320,7 @@ static const int LZ4_minLength = (MFLIMIT+1);
# endif
#endif
-#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
+#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */
#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2)
# include
@@ -529,7 +531,9 @@ static void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t cons
}
}
-static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase)
+static void LZ4_putPositionOnHash(const BYTE* p, U32 h,
+ void* tableBase, tableType_t const tableType,
+ const BYTE* srcBase)
{
switch (tableType)
{
@@ -555,8 +559,16 @@ LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_
static U32 LZ4_getIndexOnHash(U32 h, const void* tableBase, tableType_t tableType)
{
LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2);
- if (tableType == byU32) { const U32* const hashTable = (const U32*) tableBase; assert(h < (1U << (LZ4_MEMORY_USAGE-2))); return hashTable[h]; }
- if (tableType == byU16) { const U16* const hashTable = (const U16*) tableBase; assert(h < (1U << (LZ4_MEMORY_USAGE-1))); return hashTable[h]; }
+ if (tableType == byU32) {
+ const U32* const hashTable = (const U32*) tableBase;
+ assert(h < (1U << (LZ4_MEMORY_USAGE-2)));
+ return hashTable[h];
+ }
+ if (tableType == byU16) {
+ const U16* const hashTable = (const U16*) tableBase;
+ assert(h < (1U << (LZ4_MEMORY_USAGE-1)));
+ return hashTable[h];
+ }
assert(0); return 0; /* forbidden case */
}
@@ -567,7 +579,9 @@ static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType
{ const U16* const hashTable = (const U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */
}
-LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, const void* tableBase, tableType_t tableType, const BYTE* srcBase)
+LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p,
+ const void* tableBase, tableType_t tableType,
+ const BYTE* srcBase)
{
U32 const h = LZ4_hashPosition(p, tableType);
return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
@@ -650,7 +664,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
/* the dictCtx currentOffset is indexed on the start of the dictionary,
* while a dictionary in the current context precedes the currentOffset */
const BYTE* dictBase = dictDirective == usingDictCtx ?
- dictionary + dictSize - dictCtx->currentOffset : /* is it possible that dictCtx->currentOffset != dictCtx->dictSize ? Yes if the dictionary context is not reset */
+ dictionary + dictSize - dictCtx->currentOffset :
dictionary + dictSize - startIndex;
BYTE* op = (BYTE*) dest;
@@ -667,7 +681,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0);
- if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */
+ if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */
/* Update context state */
if (dictDirective == usingDictCtx) {
@@ -681,7 +695,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
cctx->currentOffset += (U32)inputSize;
cctx->tableType = tableType;
- if (inputSizehashTable, tableType, base);
@@ -790,7 +804,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
/* Copy Literals */
LZ4_wildCopy(op, anchor, op+litLength);
op+=litLength;
- DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", (int)(anchor-(const BYTE*)source), litLength, (int)(ip-(const BYTE*)source));
+ DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i",
+ (int)(anchor-(const BYTE*)source), litLength, (int)(ip-(const BYTE*)source));
}
_next_match:
@@ -905,7 +920,8 @@ _next_match:
token=op++;
*token=0;
if (maybe_extMem) offset = current - matchIndex;
- DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source));
+ DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i",
+ (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source));
goto _next_match;
}
}
@@ -1649,19 +1665,25 @@ _output_error:
LZ4_FORCE_O2_GCC_PPC64LE
int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize)
{
- return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, (BYTE*)dest, NULL, 0);
+ return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize,
+ endOnInputSize, full, 0, noDict,
+ (BYTE*)dest, NULL, 0);
}
LZ4_FORCE_O2_GCC_PPC64LE
int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize)
{
- return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0);
+ return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize,
+ endOnInputSize, partial, targetOutputSize,
+ noDict, (BYTE*)dest, NULL, 0);
}
LZ4_FORCE_O2_GCC_PPC64LE
int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
{
- return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB);
+ return LZ4_decompress_generic(source, dest, 0, originalSize,
+ endOnOutputSize, full, 0, withPrefix64k,
+ (BYTE*)(dest - 64 KB), NULL, 64 KB);
}
@@ -1803,12 +1825,18 @@ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compres
* Obsolete Functions
***************************************************/
/* obsolete compression functions */
-int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); }
-int LZ4_compress(const char* source, char* dest, int inputSize) { return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); }
-int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); }
-int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); }
-int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); }
-int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) { return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); }
+int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) {
+ return LZ4_compress_default(source, dest, inputSize, maxOutputSize); }
+int LZ4_compress(const char* source, char* dest, int inputSize) {
+ return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); }
+int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) {
+ return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); }
+int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) {
+ return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); }
+int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) {
+ return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); }
+int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) {
+ return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); }
/*
These function names are deprecated and should no longer be used.
@@ -1816,8 +1844,10 @@ They are only provided here for compatibility with older user programs.
- LZ4_uncompress is totally equivalent to LZ4_decompress_fast
- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe
*/
-int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); }
-int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
+int LZ4_uncompress (const char* source, char* dest, int outputSize) {
+ return LZ4_decompress_fast(source, dest, outputSize); }
+int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) {
+ return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
/* Obsolete Streaming functions */
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 9efaca0..e80b1a1 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -93,8 +93,8 @@ extern "C" {
**************************************/
typedef size_t LZ4F_errorCode_t;
-LZ4FLIB_API unsigned LZ4F_isError(LZ4F_errorCode_t code); /**< tells if a `LZ4F_errorCode_t` function result is an error code */
-LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return error code string; useful for debugging */
+LZ4FLIB_API unsigned LZ4F_isError(LZ4F_errorCode_t code); /**< tells when a function result is an error code */
+LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return error code string; for debugging */
/*-************************************
@@ -162,13 +162,13 @@ typedef LZ4F_contentChecksum_t contentChecksum_t;
* It's not required to set all fields, as long as the structure was initially memset() to zero.
* For all fields, 0 sets it to default value */
typedef struct {
- LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
- LZ4F_blockMode_t blockMode; /* LZ4F_blockLinked, LZ4F_blockIndependent ; 0 == default */
- LZ4F_contentChecksum_t contentChecksumFlag; /* if enabled, frame is terminated with a 32-bits checksum of decompressed data ; 0 == disabled (default) */
- LZ4F_frameType_t frameType; /* read-only field : LZ4F_frame or LZ4F_skippableFrame */
- unsigned long long contentSize; /* Size of uncompressed content ; 0 == unknown */
- unsigned dictID; /* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */
- LZ4F_blockChecksum_t blockChecksumFlag; /* if enabled, each block is followed by a checksum of block's compressed data ; 0 == disabled (default) */
+ LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB; 0 == default */
+ LZ4F_blockMode_t blockMode; /* LZ4F_blockLinked, LZ4F_blockIndependent; 0 == default */
+ LZ4F_contentChecksum_t contentChecksumFlag; /* 1: frame terminated with 32-bit checksum of decompressed data; 0: disabled (default) */
+ LZ4F_frameType_t frameType; /* read-only field : LZ4F_frame or LZ4F_skippableFrame */
+ unsigned long long contentSize; /* Size of uncompressed content ; 0 == unknown */
+ unsigned dictID; /* Dictionary ID, sent by compressor to help decoder select correct dictionary; 0 == no dictID provided */
+ LZ4F_blockChecksum_t blockChecksumFlag; /* 1: each block followed by a checksum of block's compressed data; 0: disabled (default) */
} LZ4F_frameInfo_t;
/*! LZ4F_preferences_t :
@@ -177,7 +177,7 @@ typedef struct {
* All reserved fields must be set to zero. */
typedef struct {
LZ4F_frameInfo_t frameInfo;
- int compressionLevel; /* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */
+ int compressionLevel; /* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */
unsigned autoFlush; /* 1 == always flush, to reduce usage of internal buffers */
unsigned reserved[4]; /* must be zero for forward compatibility */
} LZ4F_preferences_t;
@@ -264,36 +264,44 @@ LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t*
/*! LZ4F_compressUpdate() :
* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
- * An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations.
+ * Important rule: dstCapacity MUST be large enough to ensure operation success even in worst case situations.
* This value is provided by LZ4F_compressBound().
* If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
- * LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized.
+ * LZ4F_compressUpdate() doesn't guarantee error recovery.
+ * When an error occurs, compression context must be freed or resized.
* `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
* @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
* or an error code if it fails (which can be tested using LZ4F_isError())
*/
-LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr);
+LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx,
+ void* dstBuffer, size_t dstCapacity,
+ const void* srcBuffer, size_t srcSize,
+ const LZ4F_compressOptions_t* cOptPtr);
/*! LZ4F_flush() :
* When data must be generated and sent immediately, without waiting for a block to be completely filled,
* it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.
* `dstCapacity` must be large enough to ensure the operation will be successful.
* `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default.
- * @return : number of bytes written into dstBuffer (it can be zero, which means there was no data stored within cctx)
+ * @return : nb of bytes written into dstBuffer (can be zero, when there is no data stored within cctx)
* or an error code if it fails (which can be tested using LZ4F_isError())
*/
-LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
+LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx,
+ void* dstBuffer, size_t dstCapacity,
+ const LZ4F_compressOptions_t* cOptPtr);
/*! LZ4F_compressEnd() :
* To properly finish an LZ4 frame, invoke LZ4F_compressEnd().
* It will flush whatever data remained within `cctx` (like LZ4_flush())
* and properly finalize the frame, with an endMark and a checksum.
* `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default.
- * @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled)
+ * @return : nb of bytes written into dstBuffer, necessarily >= 4 (endMark),
* or an error code if it fails (which can be tested using LZ4F_isError())
* A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.
*/
-LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
+LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx,
+ void* dstBuffer, size_t dstCapacity,
+ const LZ4F_compressOptions_t* cOptPtr);
/*-*********************************
@@ -303,7 +311,7 @@ typedef struct LZ4F_dctx_s LZ4F_dctx; /* incomplete type */
typedef LZ4F_dctx* LZ4F_decompressionContext_t; /* compatibility with previous API versions */
typedef struct {
- unsigned stableDst; /* pledge that at least 64KB+64Bytes of previously decompressed data remain unmodifed where it was decoded. This optimization skips storage operations in tmp buffers */
+ unsigned stableDst; /* pledges that last 64KB decompressed data will remain available unmodified. This optimization skips storage operations in tmp buffers. */
unsigned reserved[3]; /* must be set to zero for forward compatibility */
} LZ4F_decompressOptions_t;
@@ -316,7 +324,7 @@ typedef struct {
* The function provides a pointer to an allocated and initialized LZ4F_dctx object.
* The result is an errorCode, which can be tested using LZ4F_isError().
* dctx memory can be released using LZ4F_freeDecompressionContext();
- * The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
+ * Result of LZ4F_freeDecompressionContext() indicates current state of decompressionContext when being released.
* That is, it should be == 0 if decompression has been completed fully and correctly.
*/
LZ4FLIB_API LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version);
@@ -357,8 +365,8 @@ LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
* The function will read up to *srcSizePtr bytes from srcBuffer,
* and decompress data into dstBuffer, of capacity *dstSizePtr.
*
- * The number of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value).
- * The number of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value).
+ * The nb of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value).
+ * The nb of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value).
*
* The function does not necessarily read all input bytes, so always check value in *srcSizePtr.
* Unconsumed source data must be presented again in subsequent invocations.
--
cgit v0.12
From 46058d71aa97cf29690f7638179104985c28d354 Mon Sep 17 00:00:00 2001
From: Yann Collet
Date: Thu, 19 Apr 2018 10:50:40 -0700
Subject: modified indentation for consistency
---
lib/lz4.c | 50 +++++++++++++++++++++++++++++++++-----------------
1 file changed, 33 insertions(+), 17 deletions(-)
diff --git a/lib/lz4.c b/lib/lz4.c
index af3221c..10674a3 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1825,29 +1825,45 @@ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compres
* Obsolete Functions
***************************************************/
/* obsolete compression functions */
-int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) {
- return LZ4_compress_default(source, dest, inputSize, maxOutputSize); }
-int LZ4_compress(const char* source, char* dest, int inputSize) {
- return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); }
-int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) {
- return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); }
-int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) {
- return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); }
-int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) {
- return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); }
-int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) {
- return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); }
+int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize)
+{
+ return LZ4_compress_default(source, dest, inputSize, maxOutputSize);
+}
+int LZ4_compress(const char* source, char* dest, int inputSize)
+{
+ return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize));
+}
+int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize)
+{
+ return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1);
+}
+int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize)
+{
+ return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1);
+}
+int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int dstCapacity)
+{
+ return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, dstCapacity, 1);
+}
+int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize)
+{
+ return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1);
+}
/*
-These function names are deprecated and should no longer be used.
+These decompression functions are deprecated and should no longer be used.
They are only provided here for compatibility with older user programs.
- LZ4_uncompress is totally equivalent to LZ4_decompress_fast
- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe
*/
-int LZ4_uncompress (const char* source, char* dest, int outputSize) {
- return LZ4_decompress_fast(source, dest, outputSize); }
-int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) {
- return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
+int LZ4_uncompress (const char* source, char* dest, int outputSize)
+{
+ return LZ4_decompress_fast(source, dest, outputSize);
+}
+int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize)
+{
+ return LZ4_decompress_safe(source, dest, isize, maxOutputSize);
+}
/* Obsolete Streaming functions */
--
cgit v0.12