From 20e969e5793aa6773593df8768d068a1ae13f746 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 5 Feb 2018 15:18:00 -0800 Subject: fuzzer: added low address compression test is expected to work on linux+gcc only. --- lib/lz4hc.c | 9 +++++---- lib/lz4hc.h | 8 ++++---- tests/fuzzer.c | 46 +++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 54 insertions(+), 9 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index cface81..79cf651 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -856,16 +856,17 @@ int LZ4_resetStreamStateHC(void* state, char* inputBuffer) LZ4HC_CCtx_internal *ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */ LZ4HC_init(ctx, (const BYTE*)inputBuffer); - ctx->inputBuffer = (BYTE*)inputBuffer; + ctx->inputBuffer = inputBuffer; return 0; } -void* LZ4_createHC (char* inputBuffer) +void* LZ4_createHC (const char* inputBuffer) { LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOCATOR(1, sizeof(LZ4_streamHC_t)); if (hc4 == NULL) return NULL; /* not enough memory */ LZ4HC_init (&hc4->internal_donotuse, (const BYTE*)inputBuffer); - hc4->internal_donotuse.inputBuffer = (BYTE*)inputBuffer; + assert(sizeof(size_t) == sizeof(void*)); + hc4->internal_donotuse.inputBuffer = (void*)(size_t)inputBuffer; /* ugly hack, circumvent -Wcast-qual */ return hc4; } @@ -889,5 +890,5 @@ char* LZ4_slideInputBufferHC(void* LZ4HC_Data) { LZ4HC_CCtx_internal* const hc4 = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse; int const dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB); - return (char*)(hc4->inputBuffer + dictSize); + return (char*)(hc4->inputBuffer) + dictSize; } diff --git a/lib/lz4hc.h b/lib/lz4hc.h index a7f77f9..7a25bee 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -148,7 +148,7 @@ typedef struct const uint8_t* end; /* next block here to continue on current prefix */ const uint8_t* base; /* All index relative to this position */ const uint8_t* dictBase; /* alternate base for extDict */ - uint8_t* inputBuffer; /* deprecated */ + void* inputBuffer; /* deprecated */ uint32_t dictLimit; /* below that point, need extDict */ uint32_t lowLimit; /* below that point, no more dict */ uint32_t nextToUpdate; /* index from which to continue dictionary update */ @@ -164,7 +164,7 @@ typedef struct const unsigned char* end; /* next block here to continue on current prefix */ const unsigned char* base; /* All index relative to this position */ const unsigned char* dictBase; /* alternate base for extDict */ - unsigned char* inputBuffer; /* deprecated */ + void* inputBuffer; /* deprecated */ unsigned int dictLimit; /* below that point, need extDict */ unsigned int lowLimit; /* below that point, no more dict */ unsigned int nextToUpdate; /* index from which to continue dictionary update */ @@ -206,8 +206,8 @@ LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_co LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize); LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); -/* Deprecated Streaming functions using older model; should no longer be used */ -LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (char* inputBuffer); +/* Deprecated Streaming functions; should no longer be used */ +LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (const char* inputBuffer); LZ4_DEPRECATED("use LZ4_saveDictHC() instead") LZ4LIB_API char* LZ4_slideInputBufferHC (void* LZ4HC_Data); LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") LZ4LIB_API int LZ4_freeHC (void* LZ4HC_Data); LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); diff --git a/tests/fuzzer.c b/tests/fuzzer.c index c134fe3..9415e94 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -240,6 +240,42 @@ _overflowError: } +#ifdef __unix__ /* is expected to be triggered on linux+gcc */ + +#include /* mmap */ + +static void* FUZ_createLowAddr(size_t size) +{ + void* const lowBuff = mmap((void*)(0x1000), size, + PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0); + DISPLAYLEVEL(2, "generating low buffer at address %p \n", lowBuff); + return lowBuff; +} + +static void FUZ_freeLowAddr(void* buffer, size_t size) +{ + if (munmap(buffer, size)) { + perror("fuzzer: freeing low address buffer"); + abort(); + } +} + +#else + +static void* FUZ_createLowAddr(size_t size) +{ + return malloc(size); +} + +static void FUZ_freeLowAddr(void* buffer, size_t size) +{ + (void)size; + free(buffer); +} + +#endif + /*! FUZ_findDiff() : * find the first different byte between buff1 and buff2. * presumes buff1 != buff2. @@ -266,6 +302,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c size_t const compressedBufferSize = LZ4_compressBound(FUZ_MAX_BLOCK_SIZE); char* const compressedBuffer = (char*)malloc(compressedBufferSize); char* const decodedBuffer = (char*)malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE); + size_t const labSize = 96 KB; + void* const lowAddrBuffer = FUZ_createLowAddr(labSize); void* const stateLZ4 = malloc(LZ4_sizeofState()); void* const stateLZ4HC = malloc(LZ4_sizeofStateHC()); LZ4_stream_t LZ4dict; @@ -306,7 +344,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; int const dictSize = MIN(dictSizeRand, blockStart); int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1); - char* const block = ((char*)CNBuffer) + blockStart; + const char* block = ((char*)CNBuffer) + blockStart; const char* dict = block - dictSize; int compressedSize, HCcompressedSize; int blockContinueCompressedSize; @@ -317,6 +355,11 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_displayUpdate(cycleNb); /* Compression tests */ + if ( ((FUZ_rand(&randState) & 63) == 2) + && ((size_t)blockSize < labSize) ) { + memcpy(lowAddrBuffer, block, blockSize); + block = lowAddrBuffer; + } /* Test compression destSize */ FUZ_DISPLAYTEST; @@ -705,6 +748,7 @@ _exit: free(CNBuffer); free(compressedBuffer); free(decodedBuffer); + FUZ_freeLowAddr(lowAddrBuffer, labSize); free(stateLZ4); free(stateLZ4HC); return result; -- cgit v0.12 From c3f0ed28ffa66fd7e28ec3b6dbbe95eb0974bfef Mon Sep 17 00:00:00 2001 From: test4973 Date: Wed, 21 Mar 2018 07:14:13 -0700 Subject: added low address fuzzer tests --- tests/Makefile | 3 +- tests/fuzzer.c | 113 +++++++++++++++++++++++++++++++-------------------------- 2 files changed, 63 insertions(+), 53 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index ddc0d2e..34b8b24 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -35,7 +35,8 @@ PRGDIR := ../programs TESTDIR := versionsTest PYTHON ?= python3 -DEBUGFLAGS = -g -DLZ4_DEBUG=1 +DEBUGLEVEL?= 1 +DEBUGFLAGS = -g -DLZ4_DEBUG=$(DEBUGLEVEL) CFLAGS ?= -O3 # can select custom optimization flags. For example : CFLAGS=-O2 make CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \ -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \ diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 9415e94..2b9b926 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -34,9 +34,13 @@ #define LZ4_DISABLE_DEPRECATE_WARNINGS + /*-************************************ * Dependencies **************************************/ +#ifdef __unix__ /* must be included before platform.h for MAP_ANONYMOUS */ +# include /* mmap */ +#endif #include "platform.h" /* _CRT_SECURE_NO_WARNINGS */ #include "util.h" /* U32 */ #include @@ -242,8 +246,6 @@ _overflowError: #ifdef __unix__ /* is expected to be triggered on linux+gcc */ -#include /* mmap */ - static void* FUZ_createLowAddr(size_t size) { void* const lowBuff = mmap((void*)(0x1000), size, @@ -276,6 +278,7 @@ static void FUZ_freeLowAddr(void* buffer, size_t size) #endif + /*! FUZ_findDiff() : * find the first different byte between buff1 and buff2. * presumes buff1 != buff2. @@ -316,10 +319,18 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c # define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \ printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; } -# define FUZ_DISPLAYTEST { testNb++; g_displayLevel>=4 ? printf("%2u\b\b", testNb), fflush(stdout) : 0; } +# define FUZ_DISPLAYTEST(...) { \ + testNb++; \ + if (g_displayLevel>=4) { \ + printf("\r%4u - %2u ", seed, testNb); \ + printf(" " __VA_ARGS__); \ + printf(" "); \ + fflush(stdout); \ + } } /* init */ + DISPLAYLEVEL(2, " g_displayLevel = %u \n", g_displayLevel); if(!CNBuffer || !compressedBuffer || !decodedBuffer) { DISPLAY("Not enough memory to start fuzzer tests"); goto _output_error; @@ -362,7 +373,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c } /* Test compression destSize */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_destSize()"); { int srcSize = blockSize; int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7; char endCheck = FUZ_rand(&randState) & 255; @@ -377,7 +388,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c U32 const crcBase = XXH32(block, srcSize, 0); char const canary = FUZ_rand(&randState) & 255; FUZ_CHECKTEST((ret==0), "LZ4_compress_destSize() compression failed"); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); compressedSize = ret; decodedBuffer[srcSize] = canary; ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize); @@ -393,7 +404,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c } } /* Test compression HC destSize */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_HC_destSize()"); { int srcSize = blockSize; int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7; char const endCheck = FUZ_rand(&randState) & 255; @@ -407,14 +418,12 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_HC_destSize() result larger than dst buffer !"); FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_HC_destSize() overwrite dst buffer !"); FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_HC_destSize() fed more than src buffer !"); - DISPLAYLEVEL(5, "LZ4_compress_HC_destSize(%i): destSize : %7i/%7i; content%7i/%7i ", - compressionLevel, ret, targetSize, srcSize, blockSize); if (targetSize>0) { /* check correctness */ U32 const crcBase = XXH32(block, srcSize, 0); char const canary = FUZ_rand(&randState) & 255; FUZ_CHECKTEST((ret==0), "LZ4_compress_HC_destSize() compression failed"); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); compressedSize = ret; decodedBuffer[srcSize] = canary; ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize); @@ -430,31 +439,31 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c } } /* Test compression HC */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_HC()"); ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel); FUZ_CHECKTEST(ret==0, "LZ4_compress_HC() failed"); HCcompressedSize = ret; /* Test compression HC using external state */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC()"); ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel); FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed"); /* Test compression using external state */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_fast_extState()"); ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8); FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed"); /* Test compression */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_default()"); ret = LZ4_compress_default(block, compressedBuffer, blockSize, (int)compressedBufferSize); FUZ_CHECKTEST(ret==0, "LZ4_compress_default() failed"); compressedSize = ret; /* Decompression tests */ - /* Test decoding with output size being exactly what's necessary => must work */ - FUZ_DISPLAYTEST; + /* Test decoding with output size exactly correct => must work */ + FUZ_DISPLAYTEST("LZ4_decompress_fast() with exact output buffer"); ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize); FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space"); FUZ_CHECKTEST(ret!=compressedSize, "LZ4_decompress_fast failed : did not fully read compressed data"); @@ -462,19 +471,19 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data"); /* Test decoding with one byte missing => must fail */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("LZ4_decompress_fast() with output buffer 1-byte too short"); decodedBuffer[blockSize-1] = 0; ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small"); FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer"); /* Test decoding with one byte too much => must fail */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large"); /* Test decoding with output size exactly what's necessary => must work */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); decodedBuffer[blockSize] = 0; ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize); FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space"); @@ -484,7 +493,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); // Test decoding with more than enough output size => must work - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); decodedBuffer[blockSize] = 0; decodedBuffer[blockSize+1] = 0; ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize+1); @@ -496,14 +505,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); // Test decoding with output size being one byte too short => must fail - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); decodedBuffer[blockSize-1] = 0; ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-1); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short"); FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe overrun specified output buffer size"); // Test decoding with output size being 10 bytes too short => must fail - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); if (blockSize>10) { decodedBuffer[blockSize-10] = 0; ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-10); @@ -512,51 +521,51 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c } // Test decoding with input size being one byte too short => must fail - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize); // Test decoding with input size being one byte too large => must fail - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); decodedBuffer[blockSize] = 0; ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize+1, blockSize); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size"); // Test partial decoding with target output size being max/2 => must work - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize/2, blockSize); FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space"); // Test partial decoding with target output size being just below max => must work - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize-3, blockSize); FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space"); /* Test Compression with limited output size */ /* Test compression with output size being exactly what's necessary (should work) */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_default() with output buffer just the right size"); ret = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize); FUZ_CHECKTEST(ret==0, "LZ4_compress_default() failed despite sufficient space"); /* Test compression with output size being exactly what's necessary and external state (should work) */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_fast_extState() with output buffer just the right size"); ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, compressedSize, 1); FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed despite sufficient space"); /* Test HC compression with output size being exactly what's necessary (should work) */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_HC() with output buffer just the right size"); ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel); FUZ_CHECKTEST(ret==0, "LZ4_compress_HC() failed despite sufficient space"); /* Test HC compression with output size being exactly what's necessary (should work) */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC() with output buffer just the right size"); ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel); FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed despite sufficient space"); /* Test compression with missing bytes into output buffer => must fail */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_default() with output buffer a bit too short"); { int missingBytes = (FUZ_rand(&randState) % 0x3F) + 1; if (missingBytes >= compressedSize) missingBytes = compressedSize-1; missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ @@ -567,7 +576,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c } /* Test HC compression with missing bytes into output buffer => must fail */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_HC() with output buffer a bit too short"); { int missingBytes = (FUZ_rand(&randState) % 0x3F) + 1; if (missingBytes >= HCcompressedSize) missingBytes = HCcompressedSize-1; missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ @@ -583,7 +592,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /*-******************/ /* Compress using dictionary */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary of size %i", dictSize); { LZ4_stream_t LZ4_stream; LZ4_resetStream(&LZ4_stream); LZ4_compress_fast_continue (&LZ4_stream, dict, compressedBuffer, dictSize, (int)compressedBufferSize, 1); /* Just to fill hash tables */ @@ -592,7 +601,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c } /* Decompress with dictionary as prefix */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_decompress_fast_usingDict() with dictionary as prefix"); memcpy(decodedBuffer, dict, dictSize); ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer+dictSize, blockSize, decodedBuffer, dictSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input"); @@ -605,33 +614,33 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c } FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_decompress_safe_usingDict()"); ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); /* Compress using External dictionary */ - FUZ_DISPLAYTEST; - dict -= (FUZ_rand(&randState) & 0xF) + 1; /* Separation, so it is an ExtDict */ + FUZ_DISPLAYTEST("test LZ4_compress_fast_continue(), with non-contiguous dictionary"); + dict -= (FUZ_rand(&randState) & 0xF) + 1; /* create space, so now dictionary is an ExtDict */ if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; LZ4_loadDict(&LZ4dict, dict, dictSize); blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue failed"); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary but with an output buffer too short by one byte"); LZ4_loadDict(&LZ4dict, dict, dictSize); ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1); FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary loaded with LZ4_loadDict()"); LZ4_loadDict(&LZ4dict, dict, dictSize); ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue should work : enough size available within output buffer"); /* Decompress with dictionary as external */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); decodedBuffer[blockSize] = 0; ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input"); @@ -640,7 +649,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); decodedBuffer[blockSize] = 0; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); @@ -648,19 +657,19 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c crcCheck = XXH32(decodedBuffer, blockSize, 0); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); decodedBuffer[blockSize-1] = 0; ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)"); FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size"); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); decodedBuffer[blockSize-1] = 0; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)"); FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); { U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2; if ((U32)blockSize > missingBytes) { decodedBuffer[blockSize-missingBytes] = 0; @@ -670,7 +679,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c } } /* Compress HC using External dictionary */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */ if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; LZ4_resetStreamHC (&LZ4dictHC, compressionLevel); @@ -679,18 +688,18 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue failed"); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDict should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue should work : enough size available within output buffer"); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); decodedBuffer[blockSize] = 0; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); @@ -701,10 +710,10 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); /* Compress HC continue destSize */ - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); { int const availableSpace = (FUZ_rand(&randState) % blockSize) + 5; int consumedSize = blockSize; - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); LZ4_resetStreamHC (&LZ4dictHC, compressionLevel); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); blockContinueCompressedSize = LZ4_compress_HC_continue_destSize(&LZ4dictHC, block, compressedBuffer, &consumedSize, availableSpace); @@ -713,7 +722,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(blockContinueCompressedSize > availableSpace, "LZ4_compress_HC_continue_destSize write overflow"); FUZ_CHECKTEST(consumedSize > blockSize, "LZ4_compress_HC_continue_destSize read overflow"); - FUZ_DISPLAYTEST; + FUZ_DISPLAYTEST(); decodedBuffer[consumedSize] = 0; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, consumedSize, dict, dictSize); FUZ_CHECKTEST(ret!=consumedSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); @@ -1118,13 +1127,13 @@ int main(int argc, const char** argv) return FUZ_usage(programName); case 'v': /* verbose mode */ - argument++; g_displayLevel++; + argument++; break; case 'p': /* pause at the end */ - argument++; use_pause=1; + argument++; break; case 'i': -- cgit v0.12 From 606afdb16428347d0bf216b2c614d962018b3fc3 Mon Sep 17 00:00:00 2001 From: Kenji Chan Date: Mon, 2 Apr 2018 10:52:45 +0800 Subject: added vs2017 projects --- visual/VS2017/datagen/datagen.vcxproj | 173 ++++++++++++++++++++ visual/VS2017/frametest/frametest.vcxproj | 180 +++++++++++++++++++++ visual/VS2017/fullbench-dll/fullbench-dll.vcxproj | 184 ++++++++++++++++++++++ visual/VS2017/fullbench/fullbench.vcxproj | 180 +++++++++++++++++++++ visual/VS2017/fuzzer/fuzzer.vcxproj | 177 +++++++++++++++++++++ visual/VS2017/liblz4-dll/liblz4-dll.rc | 51 ++++++ visual/VS2017/liblz4-dll/liblz4-dll.vcxproj | 183 +++++++++++++++++++++ visual/VS2017/liblz4/liblz4.vcxproj | 179 +++++++++++++++++++++ visual/VS2017/lz4.sln | 98 ++++++++++++ 9 files changed, 1405 insertions(+) create mode 100644 visual/VS2017/datagen/datagen.vcxproj create mode 100644 visual/VS2017/frametest/frametest.vcxproj create mode 100644 visual/VS2017/fullbench-dll/fullbench-dll.vcxproj create mode 100644 visual/VS2017/fullbench/fullbench.vcxproj create mode 100644 visual/VS2017/fuzzer/fuzzer.vcxproj create mode 100644 visual/VS2017/liblz4-dll/liblz4-dll.rc create mode 100644 visual/VS2017/liblz4-dll/liblz4-dll.vcxproj create mode 100644 visual/VS2017/liblz4/liblz4.vcxproj create mode 100644 visual/VS2017/lz4.sln diff --git a/visual/VS2017/datagen/datagen.vcxproj b/visual/VS2017/datagen/datagen.vcxproj new file mode 100644 index 0000000..30e159e --- /dev/null +++ b/visual/VS2017/datagen/datagen.vcxproj @@ -0,0 +1,173 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {D745AE2F-596A-403A-9B91-81A8C6779243} + Win32Proj + datagen + $(SolutionDir)bin\$(Platform)_$(Configuration)\ + $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\ + + + + Application + true + Unicode + v141 + + + Application + true + Unicode + v141 + + + Application + false + Unicode + true + v141 + + + Application + false + Unicode + true + v141 + + + + + + + + + + + + + + + + + + + true + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + true + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + false + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + false + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + false + MultiThreadedDebug + + + Console + true + + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true + /analyze:stacksize295252 %(AdditionalOptions) + MultiThreadedDebug + + + Console + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + false + MultiThreaded + + + Console + true + true + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + true + /analyze:stacksize295252 %(AdditionalOptions) + MultiThreaded + + + Console + true + true + true + + + + + + + + + + + + + \ No newline at end of file diff --git a/visual/VS2017/frametest/frametest.vcxproj b/visual/VS2017/frametest/frametest.vcxproj new file mode 100644 index 0000000..a3a403d --- /dev/null +++ b/visual/VS2017/frametest/frametest.vcxproj @@ -0,0 +1,180 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7} + Win32Proj + frametest + $(SolutionDir)bin\$(Platform)_$(Configuration)\ + $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\ + + + + Application + true + Unicode + v141 + + + Application + true + Unicode + v141 + + + Application + false + Unicode + true + v141 + + + Application + false + Unicode + true + v141 + + + + + + + + + + + + + + + + + + + true + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + true + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + false + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + false + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + false + MultiThreadedDebug + + + Console + true + + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true + /analyze:stacksize295252 %(AdditionalOptions) + MultiThreadedDebug + + + Console + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + false + MultiThreaded + + + Console + true + true + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + true + /analyze:stacksize295252 %(AdditionalOptions) + MultiThreaded + + + Console + true + true + true + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/visual/VS2017/fullbench-dll/fullbench-dll.vcxproj b/visual/VS2017/fullbench-dll/fullbench-dll.vcxproj new file mode 100644 index 0000000..d54a8d7 --- /dev/null +++ b/visual/VS2017/fullbench-dll/fullbench-dll.vcxproj @@ -0,0 +1,184 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {13992FD2-077E-4954-B065-A428198201A9} + Win32Proj + fullbench-dll + $(SolutionDir)bin\$(Platform)_$(Configuration)\ + $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\ + + + + Application + true + Unicode + v141 + + + Application + true + Unicode + v141 + + + Application + false + Unicode + true + v141 + + + Application + false + Unicode + true + v141 + + + + + + + + + + + + + + + + + + + true + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + true + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + false + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + false + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions) + true + false + MultiThreadedDebug + + + Console + true + $(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories) + liblz4.lib;%(AdditionalDependencies) + + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions) + true + true + /analyze:stacksize295252 %(AdditionalOptions) + MultiThreadedDebug + + + Console + true + $(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories) + liblz4.lib;%(AdditionalDependencies) + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions) + false + false + MultiThreaded + + + Console + true + true + true + $(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories) + liblz4.lib;%(AdditionalDependencies) + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions) + false + true + /analyze:stacksize295252 %(AdditionalOptions) + MultiThreaded + + + Console + true + true + true + $(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories) + liblz4.lib;%(AdditionalDependencies) + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/visual/VS2017/fullbench/fullbench.vcxproj b/visual/VS2017/fullbench/fullbench.vcxproj new file mode 100644 index 0000000..54c9743 --- /dev/null +++ b/visual/VS2017/fullbench/fullbench.vcxproj @@ -0,0 +1,180 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E} + Win32Proj + fullbench + $(SolutionDir)bin\$(Platform)_$(Configuration)\ + $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\ + + + + Application + true + Unicode + v141 + + + Application + true + Unicode + v141 + + + Application + false + Unicode + true + v141 + + + Application + false + Unicode + true + v141 + + + + + + + + + + + + + + + + + + + true + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + true + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + false + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + false + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + false + MultiThreadedDebug + + + Console + true + + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true + /analyze:stacksize295252 %(AdditionalOptions) + MultiThreadedDebug + + + Console + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + false + MultiThreaded + + + Console + true + true + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + true + /analyze:stacksize295252 %(AdditionalOptions) + MultiThreaded + + + Console + true + true + true + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/visual/VS2017/fuzzer/fuzzer.vcxproj b/visual/VS2017/fuzzer/fuzzer.vcxproj new file mode 100644 index 0000000..aa6fe42 --- /dev/null +++ b/visual/VS2017/fuzzer/fuzzer.vcxproj @@ -0,0 +1,177 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {18B9F1A7-9C66-4352-898B-30804DADE0FD} + Win32Proj + fuzzer + $(SolutionDir)bin\$(Platform)_$(Configuration)\ + $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\ + + + + Application + true + Unicode + v141 + + + Application + true + Unicode + v141 + + + Application + false + Unicode + true + v141 + + + Application + false + Unicode + true + v141 + + + + + + + + + + + + + + + + + + + true + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + true + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + false + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + false + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + false + MultiThreadedDebug + + + Console + true + + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true + /analyze:stacksize295252 %(AdditionalOptions) + MultiThreadedDebug + + + Console + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + false + MultiThreaded + + + Console + true + true + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + true + /analyze:stacksize295252 %(AdditionalOptions) + MultiThreaded + + + Console + true + true + true + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/visual/VS2017/liblz4-dll/liblz4-dll.rc b/visual/VS2017/liblz4-dll/liblz4-dll.rc new file mode 100644 index 0000000..b1871fe --- /dev/null +++ b/visual/VS2017/liblz4-dll/liblz4-dll.rc @@ -0,0 +1,51 @@ +// Microsoft Visual C++ generated resource script. +// + +#include "lz4.h" /* LZ4_VERSION_STRING */ +#define APSTUDIO_READONLY_SYMBOLS +#include "verrsrc.h" +#undef APSTUDIO_READONLY_SYMBOLS + + +#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) +LANGUAGE 9, 1 + +///////////////////////////////////////////////////////////////////////////// +// +// Version +// + +VS_VERSION_INFO VERSIONINFO + FILEVERSION LZ4_VERSION_MAJOR,LZ4_VERSION_MINOR,LZ4_VERSION_RELEASE,0 + PRODUCTVERSION LZ4_VERSION_MAJOR,LZ4_VERSION_MINOR,LZ4_VERSION_RELEASE,0 + FILEFLAGSMASK VS_FFI_FILEFLAGSMASK +#ifdef _DEBUG + FILEFLAGS VS_FF_DEBUG +#else + FILEFLAGS 0x0L +#endif + FILEOS VOS_NT_WINDOWS32 + FILETYPE VFT_DLL + FILESUBTYPE VFT2_UNKNOWN +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904B0" + BEGIN + VALUE "CompanyName", "Yann Collet" + VALUE "FileDescription", "Extremely fast compression" + VALUE "FileVersion", LZ4_VERSION_STRING + VALUE "InternalName", "lz4.dll" + VALUE "LegalCopyright", "Copyright (C) 2013-2016, Yann Collet" + VALUE "OriginalFilename", "lz4.dll" + VALUE "ProductName", "LZ4" + VALUE "ProductVersion", LZ4_VERSION_STRING + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x0409, 1200 + END +END + +#endif diff --git a/visual/VS2017/liblz4-dll/liblz4-dll.vcxproj b/visual/VS2017/liblz4-dll/liblz4-dll.vcxproj new file mode 100644 index 0000000..8e7ee3b --- /dev/null +++ b/visual/VS2017/liblz4-dll/liblz4-dll.vcxproj @@ -0,0 +1,183 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {9800039D-4AAA-43A4-BB78-FEF6F4836927} + Win32Proj + liblz4-dll + $(SolutionDir)bin\$(Platform)_$(Configuration)\ + $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\ + liblz4-dll + + + + DynamicLibrary + true + Unicode + v141 + + + DynamicLibrary + true + Unicode + v141 + + + DynamicLibrary + false + Unicode + true + v141 + + + DynamicLibrary + false + Unicode + true + v141 + + + + + + + + + + + + + + + + + + + true + liblz4 + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + true + liblz4 + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + false + liblz4 + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + false + liblz4 + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + + + + Level4 + Disabled + WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions) + true + false + MultiThreadedDebug + + + true + + + + + + + Level4 + Disabled + WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions) + true + true + /analyze:stacksize295252 %(AdditionalOptions) + MultiThreadedDebug + + + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions) + false + false + MultiThreaded + + + true + true + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions) + false + true + /analyze:stacksize295252 %(AdditionalOptions) + MultiThreaded + + + true + true + true + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/visual/VS2017/liblz4/liblz4.vcxproj b/visual/VS2017/liblz4/liblz4.vcxproj new file mode 100644 index 0000000..948f7db --- /dev/null +++ b/visual/VS2017/liblz4/liblz4.vcxproj @@ -0,0 +1,179 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476} + Win32Proj + liblz4 + $(SolutionDir)bin\$(Platform)_$(Configuration)\ + $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\ + + + + StaticLibrary + true + Unicode + v141 + + + StaticLibrary + true + Unicode + v141 + + + StaticLibrary + false + Unicode + true + v141 + + + StaticLibrary + false + Unicode + true + v141 + + + + + + + + + + + + + + + + + + + true + liblz4_static + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + true + liblz4_static + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + false + liblz4_static + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + false + liblz4_static + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + + + + Level4 + Disabled + WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions) + true + false + MultiThreadedDebug + + + true + + + + + + + Level4 + Disabled + WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions) + true + true + /analyze:stacksize295252 %(AdditionalOptions) + MultiThreadedDebug + + + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions) + false + false + MultiThreaded + + + true + true + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions) + false + true + /analyze:stacksize295252 %(AdditionalOptions) + MultiThreaded + + + true + true + true + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/visual/VS2017/lz4.sln b/visual/VS2017/lz4.sln new file mode 100644 index 0000000..78f223b --- /dev/null +++ b/visual/VS2017/lz4.sln @@ -0,0 +1,98 @@ +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Express 2012 for Windows Desktop +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "lz4", "lz4\lz4.vcxproj", "{E30329AC-0057-4FE0-8FDA-7F650D398C4C}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "liblz4-dll", "liblz4-dll\liblz4-dll.vcxproj", "{9800039D-4AAA-43A4-BB78-FEF6F4836927}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "liblz4", "liblz4\liblz4.vcxproj", "{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcxproj", "{18B9F1A7-9C66-4352-898B-30804DADE0FD}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\fullbench.vcxproj", "{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "frametest", "frametest\frametest.vcxproj", "{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "datagen", "datagen\datagen.vcxproj", "{D745AE2F-596A-403A-9B91-81A8C6779243}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench-dll", "fullbench-dll\fullbench-dll.vcxproj", "{13992FD2-077E-4954-B065-A428198201A9}" + ProjectSection(ProjectDependencies) = postProject + {9800039D-4AAA-43A4-BB78-FEF6F4836927} = {9800039D-4AAA-43A4-BB78-FEF6F4836927} + EndProjectSection +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + Release|Win32 = Release|Win32 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Win32.ActiveCfg = Debug|Win32 + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Win32.Build.0 = Debug|Win32 + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|x64.ActiveCfg = Debug|x64 + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|x64.Build.0 = Debug|x64 + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Win32.ActiveCfg = Release|Win32 + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Win32.Build.0 = Release|Win32 + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|x64.ActiveCfg = Release|x64 + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|x64.Build.0 = Release|x64 + {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Debug|Win32.ActiveCfg = Debug|Win32 + {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Debug|Win32.Build.0 = Debug|Win32 + {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Debug|x64.ActiveCfg = Debug|x64 + {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Debug|x64.Build.0 = Debug|x64 + {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Release|Win32.ActiveCfg = Release|Win32 + {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Release|Win32.Build.0 = Release|Win32 + {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Release|x64.ActiveCfg = Release|x64 + {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Release|x64.Build.0 = Release|x64 + {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|Win32.ActiveCfg = Debug|Win32 + {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|Win32.Build.0 = Debug|Win32 + {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|x64.ActiveCfg = Debug|x64 + {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|x64.Build.0 = Debug|x64 + {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|Win32.ActiveCfg = Release|Win32 + {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|Win32.Build.0 = Release|Win32 + {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|x64.ActiveCfg = Release|x64 + {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|x64.Build.0 = Release|x64 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Win32.ActiveCfg = Debug|Win32 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Win32.Build.0 = Debug|Win32 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|x64.ActiveCfg = Debug|x64 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|x64.Build.0 = Debug|x64 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Win32.ActiveCfg = Release|Win32 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Win32.Build.0 = Release|Win32 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|x64.ActiveCfg = Release|x64 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|x64.Build.0 = Release|x64 + {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|Win32.ActiveCfg = Debug|Win32 + {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|Win32.Build.0 = Debug|Win32 + {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|x64.ActiveCfg = Debug|x64 + {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|x64.Build.0 = Debug|x64 + {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|Win32.ActiveCfg = Release|Win32 + {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|Win32.Build.0 = Release|Win32 + {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|x64.ActiveCfg = Release|x64 + {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|x64.Build.0 = Release|x64 + {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|Win32.ActiveCfg = Debug|Win32 + {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|Win32.Build.0 = Debug|Win32 + {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|x64.ActiveCfg = Debug|x64 + {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|x64.Build.0 = Debug|x64 + {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|Win32.ActiveCfg = Release|Win32 + {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|Win32.Build.0 = Release|Win32 + {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|x64.ActiveCfg = Release|x64 + {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|x64.Build.0 = Release|x64 + {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|Win32.ActiveCfg = Debug|Win32 + {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|Win32.Build.0 = Debug|Win32 + {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|x64.ActiveCfg = Debug|x64 + {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|x64.Build.0 = Debug|x64 + {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|Win32.ActiveCfg = Release|Win32 + {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|Win32.Build.0 = Release|Win32 + {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|x64.ActiveCfg = Release|x64 + {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|x64.Build.0 = Release|x64 + {13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.ActiveCfg = Debug|Win32 + {13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.Build.0 = Debug|Win32 + {13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.ActiveCfg = Debug|x64 + {13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.Build.0 = Debug|x64 + {13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.ActiveCfg = Release|Win32 + {13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.Build.0 = Release|Win32 + {13992FD2-077E-4954-B065-A428198201A9}.Release|x64.ActiveCfg = Release|x64 + {13992FD2-077E-4954-B065-A428198201A9}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal -- cgit v0.12 From 6d931b6a93100fe2cb4dafc032ff486c814d1fed Mon Sep 17 00:00:00 2001 From: test4973 Date: Thu, 5 Apr 2018 12:40:33 -0700 Subject: fixed lz4 compression starting at small address when using byU32 and byU16 modes --- lib/lz4.c | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++++--- tests/fuzzer.c | 2 +- 2 files changed, 68 insertions(+), 4 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index 0fdbe5e..bcebc92 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -533,6 +533,20 @@ LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_ LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); } +/* LZ4_getIndexOnHash() : + * Index of match position registered in hash table. + * hash position must be calculated by using base+index, or dictBase+index. + * Assumption 1 : only valid if tableType == byU32 or byU16. + * Assumption 2 : h is presumed valid (within limits of hash table) + */ +static U32 LZ4_getIndexOnHash(U32 h, const void* tableBase, tableType_t tableType) +{ + LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2); + if (tableType == byU32) { const U32* const hashTable = (const U32*) tableBase; assert(h < (1U << (LZ4_MEMORY_USAGE-2))); return hashTable[h]; } + if (tableType == byU16) { const U16* const hashTable = (const U16*) tableBase; assert(h < (1U << (LZ4_MEMORY_USAGE-1))); return hashTable[h]; } + assert(0); return 0; /* forbidden case */ +} + static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType_t tableType, const BYTE* srcBase) { if (tableType == byPtr) { const BYTE* const* hashTable = (const BYTE* const*) tableBase; return hashTable[h]; } @@ -598,7 +612,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( { const BYTE* ip = (const BYTE*) source; - size_t currentOffset = cctx->currentOffset; + size_t const currentOffset = cctx->currentOffset; const BYTE* base = (const BYTE*) source - currentOffset; const BYTE* lowLimit; @@ -650,7 +664,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( BYTE* token; /* Find a match */ - { const BYTE* forwardIp = ip; + if (tableType == byPtr) { + const BYTE* forwardIp = ip; unsigned step = 1; unsigned searchMatchNb = acceleration << LZ4_skipTrigger; do { @@ -687,6 +702,54 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); + + } else { /* byU32, byU16 */ + + const BYTE* forwardIp = ip; + unsigned step = 1; + unsigned searchMatchNb = acceleration << LZ4_skipTrigger; + do { + U32 const h = forwardH; + U32 const matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; + assert(ip < mflimitPlusOne); + + if (dictDirective == usingDictCtx) { + if (matchIndex < currentOffset) { + /* there was no match, try the dictionary */ + match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase); + refDelta = dictDelta; + lowLimit = dictLowLimit; + } else { + match = base + matchIndex; + refDelta = 0; + lowLimit = (const BYTE*)source; + } + } else if (dictDirective==usingExtDict) { + if (matchIndex < currentOffset) { + match = dictBase + matchIndex; + refDelta = dictDelta; + lowLimit = dictLowLimit; + } else { + match = base + matchIndex; + refDelta = 0; + lowLimit = (const BYTE*)source; + } + } else { /* single continuous memory segment */ + match = base + matchIndex; + refDelta = 0; + lowLimit = (const BYTE*)source; + } + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); + + } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) + || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) + || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); } /* Catch up */ @@ -718,7 +781,8 @@ _next_match: /* Encode MatchLength */ { unsigned matchCode; - if ((dictDirective==usingExtDict || dictDirective==usingDictCtx) && lowLimit==dictionary) { + if ( (dictDirective==usingExtDict || dictDirective==usingDictCtx) + && (lowLimit==dictionary) ) { const BYTE* limit; match += refDelta; limit = ip + (dictEnd-match); diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 2b9b926..0b7d54e 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -322,7 +322,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c # define FUZ_DISPLAYTEST(...) { \ testNb++; \ if (g_displayLevel>=4) { \ - printf("\r%4u - %2u ", seed, testNb); \ + printf("\r%4u - %2u ", cycleNb, testNb); \ printf(" " __VA_ARGS__); \ printf(" "); \ fflush(stdout); \ -- cgit v0.12 From 64a3e41acaf9e186937d32c9dd2dc104e5bc4a72 Mon Sep 17 00:00:00 2001 From: test4973 Date: Thu, 5 Apr 2018 16:38:43 -0700 Subject: changed LZ4_compress_generic() logic to use indexes (U32) instead of Ptr. byPtr is still present. --- lib/lz4.c | 170 ++++++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 100 insertions(+), 70 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index bcebc92..4b219d2 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -516,6 +516,18 @@ LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tab return LZ4_hash4(LZ4_read32(p), tableType); } +static void LZ4_putIndexOnHash(U32 index, U32 h, void* tableBase, tableType_t const tableType) +{ + switch (tableType) + { + default: /* fallthrough */ + case clearedTable: /* fallthrough */ + case byPtr: { /* illegal! */ assert(0); return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = index; return; } + case byU16: { U16* hashTable = (U16*) tableBase; assert(index < 65536); hashTable[h] = (U16)index; return; } + } +} + static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) { switch (tableType) @@ -612,8 +624,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( { const BYTE* ip = (const BYTE*) source; - size_t const currentOffset = cctx->currentOffset; - const BYTE* base = (const BYTE*) source - currentOffset; + size_t const startIndex = cctx->currentOffset; + const BYTE* base = (const BYTE*) source - startIndex; const BYTE* lowLimit; const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx; @@ -622,7 +634,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( const U32 dictSize = dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize; - const BYTE* const lowRefLimit = (const BYTE*) source - dictSize; + int const maybe_ext_memSegment = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx); + U32 const prefixIdxLimit = startIndex - dictSize; /* used when dictDirective == dictSmall */ const BYTE* const dictEnd = dictionary + dictSize; const BYTE* anchor = (const BYTE*) source; const BYTE* const iend = ip + inputSize; @@ -633,19 +646,20 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( * while a dictionary in the current context precedes the currentOffset */ const BYTE* dictBase = dictDirective == usingDictCtx ? (const BYTE*) source - dictCtx->currentOffset : - (const BYTE*) source - dictSize - currentOffset; - const ptrdiff_t dictDelta = dictionary ? dictEnd - (const BYTE*) source : 0; + (const BYTE*) source - dictSize - startIndex; const BYTE* dictLowLimit; BYTE* op = (BYTE*) dest; BYTE* const olimit = op + maxOutputSize; + U32 offset = 0; ptrdiff_t retval = 0; U32 forwardH; /* Init conditions */ if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */ + if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */ lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0); dictLowLimit = dictionary ? dictionary : lowLimit; @@ -659,7 +673,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( /* Main Loop */ for ( ; ; ) { - ptrdiff_t refDelta = 0; const BYTE* match; BYTE* token; @@ -678,82 +691,65 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( assert(ip < mflimitPlusOne); match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base); - if (dictDirective == usingDictCtx) { - if (match < (const BYTE*)source) { - /* there was no match, try the dictionary */ - match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase); - refDelta = dictDelta; - lowLimit = dictLowLimit; - } else { - refDelta = 0; - lowLimit = (const BYTE*)source; - } - } else if (dictDirective==usingExtDict) { - if (match < (const BYTE*)source) { - refDelta = dictDelta; - lowLimit = dictLowLimit; - } else { - refDelta = 0; - lowLimit = (const BYTE*)source; - } } forwardH = LZ4_hashPosition(forwardIp, tableType); LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); - } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) - || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) - || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); + } while ( LZ4_read32(match) != LZ4_read32(ip) ); } else { /* byU32, byU16 */ const BYTE* forwardIp = ip; - unsigned step = 1; unsigned searchMatchNb = acceleration << LZ4_skipTrigger; do { U32 const h = forwardH; - U32 const matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + U32 const current = forwardIp - base; + U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + assert(matchIndex <= current); ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> LZ4_skipTrigger); + assert(searchMatchNb >= (1<> LZ4_skipTrigger); if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; assert(ip < mflimitPlusOne); if (dictDirective == usingDictCtx) { - if (matchIndex < currentOffset) { + if (matchIndex < startIndex) { /* there was no match, try the dictionary */ - match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase); - refDelta = dictDelta; + matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); + match = dictBase + matchIndex; lowLimit = dictLowLimit; } else { match = base + matchIndex; - refDelta = 0; lowLimit = (const BYTE*)source; } } else if (dictDirective==usingExtDict) { - if (matchIndex < currentOffset) { + if (matchIndex < startIndex) { match = dictBase + matchIndex; - refDelta = dictDelta; lowLimit = dictLowLimit; } else { match = base + matchIndex; - refDelta = 0; lowLimit = (const BYTE*)source; } } else { /* single continuous memory segment */ match = base + matchIndex; - refDelta = 0; - lowLimit = (const BYTE*)source; } forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); + LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); + + if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue; /* match outside of valid area */ + if ((tableType != byU16) && (matchIndex+MAX_DISTANCE < current)) continue; /* too far */ + if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE); /* too_far presumed impossible with byU16 */ - } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) - || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) - || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); + if (LZ4_read32(match) == LZ4_read32(ip)) { + if (maybe_ext_memSegment) offset = current - matchIndex; + break; /* match found */ + } + + } while(1); } /* Catch up */ - while (((ip>anchor) & (match+refDelta > lowLimit)) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } + while (((ip>anchor) & (match > lowLimit)) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } /* Encode Literals */ { unsigned const litLength = (unsigned)(ip - anchor); @@ -776,7 +772,13 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( _next_match: /* Encode Offset */ - LZ4_writeLE16(op, (U16)(ip-match)); op+=2; + if (maybe_ext_memSegment) { /* static test */ + assert(offset <= MAX_DISTANCE && offset > 0); + LZ4_writeLE16(op, (U16)offset); op+=2; + } else { + assert(ip-match <= MAX_DISTANCE); + LZ4_writeLE16(op, (U16)(ip - match)); op+=2; + } /* Encode MatchLength */ { unsigned matchCode; @@ -784,7 +786,6 @@ _next_match: if ( (dictDirective==usingExtDict || dictDirective==usingDictCtx) && (lowLimit==dictionary) ) { const BYTE* limit; - match += refDelta; limit = ip + (dictEnd-match); if (limit > matchlimit) limit = matchlimit; matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); @@ -799,6 +800,8 @@ _next_match: ip += MINMATCH + matchCode; } + DEBUGLOG(2,"matchLength:%7u ", matchCode+MINMATCH); + if ( outputLimited && /* Check output buffer overflow */ (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) goto _clean_up; @@ -825,34 +828,61 @@ _next_match: /* Fill table */ LZ4_putPosition(ip-2, cctx->hashTable, tableType, base); +#if 1 /* Test next position */ - match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); - if (dictDirective == usingDictCtx) { - if (match < (const BYTE*)source) { - /* there was no match, try the dictionary */ - match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase); - refDelta = dictDelta; - lowLimit = dictLowLimit; - } else { - refDelta = 0; - lowLimit = (const BYTE*)source; + if (tableType == byPtr) { + + match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); + LZ4_putPosition(ip, cctx->hashTable, tableType, base); + if ( (match+MAX_DISTANCE >= ip) + && (LZ4_read32(match) == LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + } else { /* byU32, byU16 */ + + U32 const h = LZ4_hashPosition(ip, tableType); + U32 const current = (U32)(ip-base); + U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + assert(matchIndex < current); + if (dictDirective == usingDictCtx) { + if (match < (const BYTE*)source) { + /* there was no match, try the dictionary */ + matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); + match = dictBase + matchIndex; + } else { + match = base + matchIndex; + } + } else if (dictDirective==usingExtDict) { + if (match < (const BYTE*)source) { + match = dictBase + matchIndex; + } else { + match = base + matchIndex; + } + } else { /* single memory segment */ + match = base + matchIndex; } - } else if (dictDirective==usingExtDict) { - if (match < (const BYTE*)source) { - refDelta = dictDelta; - lowLimit = dictLowLimit; - } else { - refDelta = 0; - lowLimit = (const BYTE*)source; - } } - LZ4_putPosition(ip, cctx->hashTable, tableType, base); - if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) - && (match+MAX_DISTANCE>=ip) - && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) - { token=op++; *token=0; goto _next_match; } + LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); + if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1) + && ((tableType==byU16) ? 1 : (matchIndex+MAX_DISTANCE >= current)) + && (LZ4_read32(match) == LZ4_read32(ip)) ) { + token=op++; + *token=0; + if (maybe_ext_memSegment) + offset = current - matchIndex; + goto _next_match; + } + } /* Prepare next loop */ forwardH = LZ4_hashPosition(++ip, tableType); + +#else + + /* Prepare next loop */ + forwardH = LZ4_hashPosition(ip, tableType); + +#endif + } _last_literals: -- cgit v0.12 From f2a4d6ef37f653c21627274634d171af66126d5e Mon Sep 17 00:00:00 2001 From: test4973 Date: Thu, 5 Apr 2018 17:16:33 -0700 Subject: fixed immediate match search --- lib/lz4.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index 4b219d2..8d8c1e8 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -624,7 +624,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( { const BYTE* ip = (const BYTE*) source; - size_t const startIndex = cctx->currentOffset; + U32 const startIndex = cctx->currentOffset; const BYTE* base = (const BYTE*) source - startIndex; const BYTE* lowLimit; @@ -645,8 +645,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( /* the dictCtx currentOffset is indexed on the start of the dictionary, * while a dictionary in the current context precedes the currentOffset */ const BYTE* dictBase = dictDirective == usingDictCtx ? - (const BYTE*) source - dictCtx->currentOffset : - (const BYTE*) source - dictSize - startIndex; + dictionary + dictSize - dictCtx->currentOffset : /* is it possible that dictCtx->currentOffset != dictCtx->dictSize ? */ + dictionary + dictSize - startIndex; const BYTE* dictLowLimit; BYTE* op = (BYTE*) dest; @@ -800,8 +800,6 @@ _next_match: ip += MINMATCH + matchCode; } - DEBUGLOG(2,"matchLength:%7u ", matchCode+MINMATCH); - if ( outputLimited && /* Check output buffer overflow */ (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) goto _clean_up; @@ -845,7 +843,7 @@ _next_match: U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); assert(matchIndex < current); if (dictDirective == usingDictCtx) { - if (match < (const BYTE*)source) { + if (matchIndex < startIndex) { /* there was no match, try the dictionary */ matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); match = dictBase + matchIndex; @@ -853,7 +851,7 @@ _next_match: match = base + matchIndex; } } else if (dictDirective==usingExtDict) { - if (match < (const BYTE*)source) { + if (matchIndex < startIndex) { match = dictBase + matchIndex; } else { match = base + matchIndex; -- cgit v0.12 From b4be1e0a743f2200eaf1c13d322c925b64b872e2 Mon Sep 17 00:00:00 2001 From: test4973 Date: Thu, 5 Apr 2018 17:52:54 -0700 Subject: fixed byPtr match search --- lib/lz4.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/lz4.c b/lib/lz4.c index 8d8c1e8..5791556 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -694,7 +694,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( forwardH = LZ4_hashPosition(forwardIp, tableType); LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); - } while ( LZ4_read32(match) != LZ4_read32(ip) ); + } while ( (match+MAX_DISTANCE < ip) + || (LZ4_read32(match) != LZ4_read32(ip)) ); } else { /* byU32, byU16 */ -- cgit v0.12 From f4e06e28e6d285f7f145798d7dfe1cbe71ae1efa Mon Sep 17 00:00:00 2001 From: test4973 Date: Thu, 5 Apr 2018 18:29:42 -0700 Subject: fixed byPtr mode switch to byU32 when src address is < 64K note : byPtr is still useful in 32-bits, as it's about ~10% faster --- lib/lz4.c | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index 5791556..4fb54f8 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -930,14 +930,14 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int if (inputSize < LZ4_64Klimit) { return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); } else { - const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr; + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > MAX_DISTANCE)) ? byPtr : byU32; return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration); } } else { if (inputSize < LZ4_64Klimit) {; return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); } else { - const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr; + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > MAX_DISTANCE)) ? byPtr : byU32; return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration); } } @@ -951,7 +951,7 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int * performance when the context reset would otherwise be a significant part of * the cost of the compression, e.g., when the data to be compressed is small. */ -int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +int LZ4_compress_fast_extState_noReset(void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration) { LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse; if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; @@ -959,39 +959,39 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de ctx->dictSize = 0; ctx->dictCtx = NULL; - if (maxOutputSize >= LZ4_compressBound(inputSize)) { - if (inputSize < LZ4_64Klimit) { + if (dstCapacity >= LZ4_compressBound(srcSize)) { + if (srcSize < LZ4_64Klimit) { const tableType_t tableType = byU16; - LZ4_prepareTable(ctx, inputSize, tableType, noDict); + LZ4_prepareTable(ctx, srcSize, tableType, noDict); if (ctx->currentOffset) { - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, dictSmall, acceleration); + return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, dictSmall, acceleration); } else { - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration); } } else { - const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr; - LZ4_prepareTable(ctx, inputSize, tableType, noDict); + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32; + LZ4_prepareTable(ctx, srcSize, tableType, noDict); if (ctx->currentOffset) { ctx->currentOffset += 64 KB; } - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration); } } else { - if (inputSize < LZ4_64Klimit) { + if (srcSize < LZ4_64Klimit) { const tableType_t tableType = byU16; - LZ4_prepareTable(ctx, inputSize, tableType, noDict); + LZ4_prepareTable(ctx, srcSize, tableType, noDict); if (ctx->currentOffset) { - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration); + return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration); } else { - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); } } else { - const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr; - LZ4_prepareTable(ctx, inputSize, tableType, noDict); + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32; + LZ4_prepareTable(ctx, srcSize, tableType, noDict); if (ctx->currentOffset) { ctx->currentOffset += 64 KB; } - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); } } } @@ -1098,6 +1098,8 @@ static int LZ4_compress_destSize_generic( forwardH = LZ4_hashPosition(forwardIp, tableType); LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base); + DEBUGLOG(2, "match:%p , ip:%p", match, ip); + } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) || (LZ4_read32(match) != LZ4_read32(ip)) ); } @@ -1203,11 +1205,12 @@ static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src, if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); } else { - if (*srcSizePtr < LZ4_64Klimit) + if (*srcSizePtr < LZ4_64Klimit) { return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16); - else - return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, sizeof(void*)==8 ? byU32 : byPtr); - } + } else { + tableType_t const tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32; + return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, tableType); + } } } -- cgit v0.12 From 038a0d95bfe2d3a544aa2f5551998f6fd8bc0722 Mon Sep 17 00:00:00 2001 From: test4973 Date: Thu, 5 Apr 2018 18:39:22 -0700 Subject: added low-memory address test to travis requires modification linux configuration (sudo) --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 466d55e..a446420 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ matrix: - os: linux sudo: false - env: Ubu=12.04cont Cmd='make -C tests test-frametest test-fuzzer' COMPILER=cc + env: Ubu=12.04cont Cmd='sudo sysctl -w vm.mmap_min_addr="4096" && make -C tests test-frametest test-fuzzer' COMPILER=cc - os: linux sudo: false @@ -59,7 +59,7 @@ matrix: - libc6-dev-i386 - gcc-multilib - - env: Ubu=14.04 Cmd='make -C tests test-frametest32 test-fuzzer32' COMPILER=cc + - env: Ubu=14.04 Cmd='sudo sysctl -w vm.mmap_min_addr="4096" && make -C tests test-frametest32 test-fuzzer32' COMPILER=cc dist: trusty sudo: required addons: -- cgit v0.12 From f9992fa37f1b0810c4d0a3e3e6a0eb4880168c57 Mon Sep 17 00:00:00 2001 From: test4973 Date: Thu, 5 Apr 2018 19:05:49 -0700 Subject: noticed a bug when re-using hash table ./fuzzer -vv -s4217 -t7518 --- lib/lz4.c | 7 ++++--- tests/fuzzer.c | 59 +++++++++++++++++++++++++++++----------------------------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index 4fb54f8..54336c7 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -593,6 +593,9 @@ LZ4_FORCE_INLINE void LZ4_prepareTable( MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE); cctx->currentOffset = 0; cctx->tableType = clearedTable; + } else { + DEBUGLOG(4, "Re-use hash table (no reset)"); + //if (tableType == byU32) cctx->currentOffset += 64 KB; } } /* If the current offset is zero, we will never look in the external @@ -1098,8 +1101,6 @@ static int LZ4_compress_destSize_generic( forwardH = LZ4_hashPosition(forwardIp, tableType); LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base); - DEBUGLOG(2, "match:%p , ip:%p", match, ip); - } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) || (LZ4_read32(match) != LZ4_read32(ip)) ); } @@ -1371,7 +1372,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch LZ4_prepareTable(streamPtr, inputSize, tableType, usingDictCtx); result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration); } - } else { + } else { /* no dictCtx */ LZ4_prepareTable(streamPtr, inputSize, tableType, usingExtDict); if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration); diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 0b7d54e..8ffdb22 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -291,7 +291,7 @@ static void FUZ_findDiff(const void* buff1, const void* buff2) const BYTE* const b2 = (const BYTE*)buff2; size_t u = 0; while (b1[u]==b2[u]) u++; - DISPLAY("Wrong Byte at position %u \n", (unsigned)u); + DISPLAY("\nWrong Byte at position %u \n", (unsigned)u); } @@ -330,7 +330,6 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* init */ - DISPLAYLEVEL(2, " g_displayLevel = %u \n", g_displayLevel); if(!CNBuffer || !compressedBuffer || !decodedBuffer) { DISPLAY("Not enough memory to start fuzzer tests"); goto _output_error; @@ -360,7 +359,6 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c int compressedSize, HCcompressedSize; int blockContinueCompressedSize; U32 const crcOrig = XXH32(block, blockSize, 0); - U32 crcCheck; int ret; FUZ_displayUpdate(cycleNb); @@ -467,8 +465,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize); FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space"); FUZ_CHECKTEST(ret!=compressedSize, "LZ4_decompress_fast failed : did not fully read compressed data"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data"); + { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data"); + } /* Test decoding with one byte missing => must fail */ FUZ_DISPLAYTEST("LZ4_decompress_fast() with output buffer 1-byte too short"); @@ -489,8 +488,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space"); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); + { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); + } // Test decoding with more than enough output size => must work FUZ_DISPLAYTEST(); @@ -501,8 +501,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data"); //FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe wrote more than (unknown) target size"); // well, is that an issue ? FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ4_decompress_safe overrun specified output buffer size"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); + { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); + } // Test decoding with output size being one byte too short => must fail FUZ_DISPLAYTEST(); @@ -605,20 +606,17 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c memcpy(decodedBuffer, dict, dictSize); ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer+dictSize, blockSize, decodedBuffer, dictSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input"); - crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); - if (crcCheck!=crcOrig) { - int i=0; - while (block[i]==decodedBuffer[i]) i++; - printf("Wrong Byte at position %i/%i\n", i, blockSize); - + { U32 const crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); + if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); } - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); FUZ_DISPLAYTEST("test LZ4_decompress_safe_usingDict()"); ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); - crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); + { U32 const crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); + } /* Compress using External dictionary */ FUZ_DISPLAYTEST("test LZ4_compress_fast_continue(), with non-contiguous dictionary"); @@ -640,22 +638,24 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue should work : enough size available within output buffer"); /* Decompress with dictionary as external */ - FUZ_DISPLAYTEST(); + FUZ_DISPLAYTEST("test LZ4_decompress_fast_usingDict(): decoding %i bytes, dict(%p) of size %i", blockSize, dict, dictSize); decodedBuffer[blockSize] = 0; ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); + { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); + } FUZ_DISPLAYTEST(); decodedBuffer[blockSize] = 0; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); + { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); + } FUZ_DISPLAYTEST(); decodedBuffer[blockSize-1] = 0; @@ -704,10 +704,10 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - if (crcCheck!=crcOrig) - FUZ_findDiff(block, decodedBuffer); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); + { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); + } /* Compress HC continue destSize */ FUZ_DISPLAYTEST(); @@ -729,8 +729,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(decodedBuffer[consumedSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size") { U32 const crcSrc = XXH32(block, consumedSize, 0); U32 const crcDst = XXH32(decodedBuffer, consumedSize, 0); - if (crcSrc!=crcDst) - FUZ_findDiff(block, decodedBuffer); + if (crcSrc!=crcDst) FUZ_findDiff(block, decodedBuffer); FUZ_CHECKTEST(crcSrc!=crcDst, "LZ4_decompress_safe_usingDict corrupted decoded data"); } } -- cgit v0.12 From 133a50b780be302e726da772592e505d5b143f44 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 6 Apr 2018 14:16:23 -0700 Subject: fixed DISPLAYUPDATE() wrong comparison, which was always overflowing (hence was always true) except when it was not (i386, reported by pmc) in which case it would never show any information. --- programs/lz4io.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index ca13316..6d0d0d0 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -94,9 +94,12 @@ static int g_displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */ #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ - if (((clock_t)(g_time - clock()) > refreshRate) || (g_displayLevel>=4)) \ - { g_time = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stderr); } } + if ( ((clock() - g_time) > refreshRate) \ + || (g_displayLevel>=4) ) { \ + g_time = clock(); \ + DISPLAY(__VA_ARGS__); \ + if (g_displayLevel>=4) fflush(stderr); \ + } } static const clock_t refreshRate = CLOCKS_PER_SEC / 6; static clock_t g_time = 0; -- cgit v0.12 From 5622c276e114c75f11f6171f9fe2295a8b626e89 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 6 Apr 2018 18:52:55 -0400 Subject: Return to Allowing Early Returns in LZ4_compress_generic() Or: `goto` Considered Harmful Or: https://xkcd.com/292/ --- lib/lz4.c | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index 0fdbe5e..8743cb9 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -626,8 +626,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( BYTE* op = (BYTE*) dest; BYTE* const olimit = op + maxOutputSize; - ptrdiff_t retval = 0; - U32 forwardH; /* Init conditions */ @@ -637,6 +635,19 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( dictLowLimit = dictionary ? dictionary : lowLimit; if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ + + /* Update context state */ + if (dictDirective == usingDictCtx) { + /* Subsequent linked blocks can't use the dictionary. */ + /* Instead, they use the block we just compressed. */ + cctx->dictCtx = NULL; + cctx->dictSize = (U32)inputSize; + } else { + cctx->dictSize += (U32)inputSize; + } + cctx->currentOffset += (U32)inputSize; + cctx->tableType = tableType; + if (inputSize olimit))) - goto _clean_up; + return 0; if (litLength >= RUN_MASK) { int len = (int)litLength-RUN_MASK; *token = (RUN_MASK<>8) > olimit)) ) - goto _clean_up; + return 0; if (matchCode >= ML_MASK) { *token += ML_MASK; matchCode -= ML_MASK; @@ -796,7 +807,7 @@ _last_literals: { size_t const lastRun = (size_t)(iend - anchor); if ( (outputLimited) && /* Check output buffer overflow */ ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) ) - goto _clean_up; + return 0; if (lastRun >= RUN_MASK) { size_t accumulator = lastRun - RUN_MASK; *op++ = RUN_MASK << ML_BITS; @@ -809,22 +820,7 @@ _last_literals: op += lastRun; } - retval = (((char*)op)-dest); - -_clean_up: - if (dictDirective == usingDictCtx) { - /* Subsequent linked blocks can't use the dictionary. */ - /* Instead, they use the block we just compressed. */ - cctx->dictCtx = NULL; - cctx->dictSize = (U32)inputSize; - } else { - cctx->dictSize += (U32)inputSize; - } - cctx->currentOffset += (U32)inputSize; - cctx->tableType = tableType; - - /* End */ - return (int)retval; + return (int)(((char*)op) - dest); } -- cgit v0.12 From f88dc900553b3a029e9cd114800016f9364ffc7b Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 6 Apr 2018 16:52:29 -0400 Subject: Avoid Calling LZ4_prepareTable() in LZ4_compress_fast_continue() --- lib/lz4.c | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index 8743cb9..4f353c5 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -574,11 +574,16 @@ LZ4_FORCE_INLINE void LZ4_prepareTable( * indicates a miss. In that case, we need to bump the offset to something * non-zero. */ - if (dictDirective == usingDictCtx && - tableType != byPtr && - cctx->currentOffset == 0) - { - cctx->currentOffset = 1; + if (cctx->currentOffset == 0) { + if (dictDirective == usingDictCtx) { + if (tableType == byU16) { + cctx->currentOffset = 1; + } else if (tableType == byU32) { + cctx->currentOffset = 64 KB; + } + } + } else if (tableType == byU32) { + cctx->currentOffset += 64 KB; } } @@ -874,9 +879,6 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de } else { const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr; LZ4_prepareTable(ctx, inputSize, tableType, noDict); - if (ctx->currentOffset) { - ctx->currentOffset += 64 KB; - } return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration); } } else { @@ -891,9 +893,6 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de } else { const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr; LZ4_prepareTable(ctx, inputSize, tableType, noDict); - if (ctx->currentOffset) { - ctx->currentOffset += 64 KB; - } return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration); } } @@ -1168,31 +1167,28 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_stream) int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) { LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse; + const tableType_t tableType = byU32; const BYTE* p = (const BYTE*)dictionary; const BYTE* const dictEnd = p + dictSize; const BYTE* base; DEBUGLOG(4, "LZ4_loadDict %p", LZ4_dict); - if ((dict->initCheck) - || (dict->tableType != byU32 && dict->tableType != clearedTable) - || (dict->currentOffset > 1 GB)) /* Uninitialized structure, or reuse overflow */ - LZ4_resetStream(LZ4_dict); + LZ4_prepareTable(dict, 0, tableType, usingExtDict); if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; - dict->currentOffset += 64 KB; base = p - dict->currentOffset; dict->dictionary = p; dict->dictSize = (U32)(dictEnd - p); dict->currentOffset += dict->dictSize; - dict->tableType = byU32; + dict->tableType = tableType; if (dictSize < (int)HASH_UNIT) { return 0; } while (p <= dictEnd-HASH_UNIT) { - LZ4_putPosition(p, dict->hashTable, byU32, base); + LZ4_putPosition(p, dict->hashTable, tableType, base); p+=3; } @@ -1244,7 +1240,6 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch /* prefix mode : source data follows dictionary */ if (dictEnd == (const BYTE*)source) { - LZ4_prepareTable(streamPtr, inputSize, tableType, withPrefix64k); if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration); else @@ -1272,7 +1267,6 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration); } } else { - LZ4_prepareTable(streamPtr, inputSize, tableType, usingExtDict); if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration); } else { -- cgit v0.12 From cf2f06a6c5046ef7d576bc8ad210c3f2efe1f401 Mon Sep 17 00:00:00 2001 From: test4973 Date: Mon, 9 Apr 2018 17:08:17 -0700 Subject: fixed minor conversion warning ptr diff -> U32 --- lib/lz4.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/lz4.c b/lib/lz4.c index 3db37b0..a3c2860 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -717,9 +717,10 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( unsigned searchMatchNb = acceleration << LZ4_skipTrigger; do { U32 const h = forwardH; - U32 const current = forwardIp - base; + U32 const current = (U32)(forwardIp - base); U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); assert(matchIndex <= current); + assert(forwardIp - base < (ptrdiff_t)(2 GB - 1)); ip = forwardIp; assert(searchMatchNb >= (1<> LZ4_skipTrigger); -- cgit v0.12 From ad7e040384d1835c097b0f727aeaa3d598a401b6 Mon Sep 17 00:00:00 2001 From: test4973 Date: Mon, 9 Apr 2018 20:38:00 -0700 Subject: fix minor conversion warning cast from void not implicit for C++ --- tests/fuzzer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 8ffdb22..2e3ee92 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -367,7 +367,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c if ( ((FUZ_rand(&randState) & 63) == 2) && ((size_t)blockSize < labSize) ) { memcpy(lowAddrBuffer, block, blockSize); - block = lowAddrBuffer; + block = (const char*)lowAddrBuffer; } /* Test compression destSize */ -- cgit v0.12 From 59c7d951213bf8593438ae811549655d5a78d2fc Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 10 Apr 2018 13:12:30 -0400 Subject: Expose a Faster Stream Reset Function --- lib/lz4.c | 53 ++++++++++++++++++++++++++++------------------------- lib/lz4.h | 8 ++++++++ lib/lz4frame.c | 4 +--- 3 files changed, 37 insertions(+), 28 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index 4f353c5..30146bb 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -546,12 +546,10 @@ LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, const void* tableBas return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); } - LZ4_FORCE_INLINE void LZ4_prepareTable( LZ4_stream_t_internal* const cctx, const int inputSize, - const tableType_t tableType, - const dict_directive dictDirective) { + const tableType_t tableType) { /* If the table hasn't been used, it's guaranteed to be zeroed out, and is * therefore safe to use no matter what mode we're in. Otherwise, we figure * out if it's safe to leave as is or whether it needs to be reset. @@ -569,22 +567,23 @@ LZ4_FORCE_INLINE void LZ4_prepareTable( cctx->tableType = clearedTable; } } - /* If the current offset is zero, we will never look in the external - * dictionary context, since there is no value a table entry can take that - * indicates a miss. In that case, we need to bump the offset to something - * non-zero. + + /* Adding a gap, so all previous entries are > MAX_DISTANCE back, is faster + * than compressing without a gap. However, compressing with + * currentOffset == 0 is faster still, so we preserve that case. */ - if (cctx->currentOffset == 0) { - if (dictDirective == usingDictCtx) { - if (tableType == byU16) { - cctx->currentOffset = 1; - } else if (tableType == byU32) { - cctx->currentOffset = 64 KB; - } - } - } else if (tableType == byU32) { + if (cctx->currentOffset != 0 && tableType == byU32) { cctx->currentOffset += 64 KB; } + + /* Finally, clear history */ + cctx->dictCtx = NULL; + cctx->dictionary = NULL; + cctx->dictSize = 0; +} + +void LZ4_resetStream_fast(LZ4_stream_t* const ctx) { + LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32); } /** LZ4_compress_generic() : @@ -863,14 +862,11 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de { LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse; if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; - ctx->dictionary = NULL; - ctx->dictSize = 0; - ctx->dictCtx = NULL; if (maxOutputSize >= LZ4_compressBound(inputSize)) { if (inputSize < LZ4_64Klimit) { const tableType_t tableType = byU16; - LZ4_prepareTable(ctx, inputSize, tableType, noDict); + LZ4_prepareTable(ctx, inputSize, tableType); if (ctx->currentOffset) { return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, dictSmall, acceleration); } else { @@ -878,13 +874,13 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de } } else { const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr; - LZ4_prepareTable(ctx, inputSize, tableType, noDict); + LZ4_prepareTable(ctx, inputSize, tableType); return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration); } } else { if (inputSize < LZ4_64Klimit) { const tableType_t tableType = byU16; - LZ4_prepareTable(ctx, inputSize, tableType, noDict); + LZ4_prepareTable(ctx, inputSize, tableType); if (ctx->currentOffset) { return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration); } else { @@ -892,7 +888,7 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de } } else { const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr; - LZ4_prepareTable(ctx, inputSize, tableType, noDict); + LZ4_prepareTable(ctx, inputSize, tableType); return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration); } } @@ -1174,7 +1170,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) DEBUGLOG(4, "LZ4_loadDict %p", LZ4_dict); - LZ4_prepareTable(dict, 0, tableType, usingExtDict); + LZ4_prepareTable(dict, 0, tableType); if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; base = p - dict->currentOffset; @@ -1263,7 +1259,14 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t)); result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); } else { - LZ4_prepareTable(streamPtr, inputSize, tableType, usingDictCtx); + /* If the current offset is zero, we will never look in the + * external dictionary context, since there is no value a table + * entry can take that indicate a miss. In that case, we need + * to bump the offset to something non-zero. + */ + if (streamPtr->currentOffset == 0) { + streamPtr->currentOffset = 64 KB; + } result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration); } } else { diff --git a/lib/lz4.h b/lib/lz4.h index 80f040f..9c1b694 100644 --- a/lib/lz4.h +++ b/lib/lz4.h @@ -366,6 +366,14 @@ LZ4_compress_fast_extState_noReset() : LZ4LIB_API int LZ4_compress_fast_extState_noReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + +/*! LZ4_resetStream_fast() : + * An LZ4_stream_t structure can be allocated once and re-used multiple times. + * Use this function to start compressing a new stream. + */ +LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr); + + /*-************************************ * Private definitions ************************************** diff --git a/lib/lz4frame.c b/lib/lz4frame.c index a080fa6..afa5af3 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -532,9 +532,7 @@ static void LZ4F_applyCDict(void* ctx, if (level < LZ4HC_CLEVEL_MIN) { LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t *)ctx)->internal_donotuse; assert(!internal_ctx->initCheck); - /* Clear any local dictionary */ - internal_ctx->dictionary = NULL; - internal_ctx->dictSize = 0; + LZ4_resetStream_fast((LZ4_stream_t *)ctx); /* Point to the dictionary context */ internal_ctx->dictCtx = cdict ? &(cdict->fastCtx->internal_donotuse) : NULL; } else { -- cgit v0.12 From c18bff933b18106ea000868ce13f37f0589cb58e Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 11 Apr 2018 15:12:34 -0400 Subject: Remove Extraneous Assignment (clearedTable == 0) --- lib/lz4.c | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/lz4.c b/lib/lz4.c index 30146bb..f79ee2c 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -1147,7 +1147,6 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream) { DEBUGLOG(5, "LZ4_resetStream %p", LZ4_stream); MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); - LZ4_stream->internal_donotuse.tableType = clearedTable; } int LZ4_freeStream (LZ4_stream_t* LZ4_stream) -- cgit v0.12 From 21f0c9700b28301bdc8b49fcbe4ccdddf4cf5dea Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 11 Apr 2018 15:13:01 -0400 Subject: Rename _extState_noReset -> _extState_fastReset and Edit Comments --- lib/lz4.c | 23 ++++++++++++----------- lib/lz4.h | 43 ++++++++++++++++++++++++++++--------------- lib/lz4frame.c | 2 +- 3 files changed, 41 insertions(+), 27 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index f79ee2c..c2d1c32 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -582,10 +582,6 @@ LZ4_FORCE_INLINE void LZ4_prepareTable( cctx->dictSize = 0; } -void LZ4_resetStream_fast(LZ4_stream_t* const ctx) { - LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32); -} - /** LZ4_compress_generic() : inlined, to ensure branches are decided at compilation time */ LZ4_FORCE_INLINE int LZ4_compress_generic( @@ -851,14 +847,15 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int } /** - * LZ4_compress_fast_extState_noReset is a variant of LZ4_compress_fast_extState - * that can be used when the state is known to have already been initialized - * (via LZ4_resetStream or an earlier call to LZ4_compress_fast_extState / - * LZ4_compress_fast_extState_noReset). This can provide significantly better - * performance when the context reset would otherwise be a significant part of - * the cost of the compression, e.g., when the data to be compressed is small. + * LZ4_compress_fast_extState_fastReset() : + * A variant of LZ4_compress_fast_extState(). + * + * Using this variant avoids an expensive initialization step. It is only safe + * to call if the state buffer is known to be correctly initialized already + * (see comment in lz4.h on LZ4_resetStream_fast() for a definition of + * "correctly initialized"). */ -int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +int LZ4_compress_fast_extState_fastReset(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse; if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; @@ -1149,6 +1146,10 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream) MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); } +void LZ4_resetStream_fast(LZ4_stream_t* const ctx) { + LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32); +} + int LZ4_freeStream (LZ4_stream_t* LZ4_stream) { if (!LZ4_stream) return 0; /* support free on NULL */ diff --git a/lib/lz4.h b/lib/lz4.h index 9c1b694..427beaf 100644 --- a/lib/lz4.h +++ b/lib/lz4.h @@ -353,26 +353,39 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or * Their signatures may change. **************************************/ -/*! -LZ4_compress_fast_extState_noReset() : - A variant of LZ4_compress_fast_extState(). - - Use the _noReset variant if LZ4_resetStream() was called on the state - buffer before being used for the first time (calls to both extState - functions leave the state in a safe state, so zeroing is not required - between calls). Otherwise, using the plain _extState requires LZ4 to - reinitialize the state internally for every call. -*/ -LZ4LIB_API int LZ4_compress_fast_extState_noReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); - - /*! LZ4_resetStream_fast() : - * An LZ4_stream_t structure can be allocated once and re-used multiple times. - * Use this function to start compressing a new stream. + * When an LZ4_stream_t is known to be in a internally coherent state, + * it can often be prepared for a new compression with almost no work, only + * sometimes falling back to the full, expensive reset that is always required + * when the stream is in an indeterminate state (i.e., the reset performed by + * LZ4_resetStream()). + * + * LZ4_streams are guaranteed to be in a valid state when: + * - returned from LZ4_createStream() + * - reset by LZ4_resetStream() + * - memset(stream, 0, sizeof(LZ4_stream_t)) + * - the stream was in a valid state and was reset by LZ4_resetStream_fast() + * - the stream was in a valid state and was then used in any compression call + * that returned success + * - the stream was in an indeterminate state and was used in a compression + * call that fully reset the state (LZ4_compress_fast_extState()) and that + * returned success */ LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr); +/*! LZ4_compress_fast_extState_fastReset() : + * A variant of LZ4_compress_fast_extState(). + * + * Using this variant avoids an expensive initialization step. It is only safe + * to call if the state buffer is known to be correctly initialized already + * (see above comment on LZ4_resetStream_fast() for a definition of "correctly + * initialized"). From a high level, the difference is that this function + * initializes the provided state with a call to LZ4_resetStream_fast() while + * LZ4_compress_fast_extState() starts with a call to LZ4_resetStream(). + */ +LZ4LIB_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + /*-************************************ * Private definitions diff --git a/lib/lz4frame.c b/lib/lz4frame.c index afa5af3..5e11ba2 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -714,7 +714,7 @@ static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize if (cdict) { return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration); } else { - return LZ4_compress_fast_extState_noReset(ctx, src, dst, srcSize, dstCapacity, acceleration); + return LZ4_compress_fast_extState_fastReset(ctx, src, dst, srcSize, dstCapacity, acceleration); } } -- cgit v0.12 From afa52c9b95dd1f83d80cd9a783a408822700e2d3 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 11 Apr 2018 16:04:24 -0400 Subject: Expose dictCtx Functionality in LZ4 --- lib/lz4.c | 4 ++++ lib/lz4.h | 28 ++++++++++++++++++++++++++++ lib/lz4frame.c | 3 +-- 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index c2d1c32..cee85ec 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -1191,6 +1191,10 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) return dict->dictSize; } +void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream) { + working_stream->internal_donotuse.dictCtx = dictionary_stream != NULL ? &(dictionary_stream->internal_donotuse) : NULL; +} + static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) { diff --git a/lib/lz4.h b/lib/lz4.h index 427beaf..f90120e 100644 --- a/lib/lz4.h +++ b/lib/lz4.h @@ -386,6 +386,34 @@ LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr); */ LZ4LIB_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); +/*! LZ4_attach_dictionary() : + * This is an experimental API that allows for the efficient use of a + * static dictionary many times. + * + * Rather than re-loading the dictionary buffer into a working context before + * each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a + * working LZ4_stream_t, this function introduces a no-copy setup mechanism, + * in which the working stream references the dictionary stream in-place. + * + * Several assumptions are made about the state of the dictionary stream. + * Currently, only streams which have been prepared by LZ4_loadDict() should + * be expected to work. + * + * Alternatively, the provided dictionary stream pointer may be NULL, in which + * case any existing dictionary stream is unset. + * + * If a dictionary is provided, it replaces any pre-existing stream history. + * The dictionary contents are the only history that can be referenced and + * logically immediately precede the data compressed in the first subsequent + * compression call. + * + * The dictionary will only remain attached to the working stream through the + * first compression call, at the end of which it is cleared. The dictionary + * stream (and source buffer) must remain in-place / accessible / unchanged + * through the completion of the first compression call on the stream. + */ +LZ4LIB_API void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream); + /*-************************************ * Private definitions diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 5e11ba2..d973b5f 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -533,8 +533,7 @@ static void LZ4F_applyCDict(void* ctx, LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t *)ctx)->internal_donotuse; assert(!internal_ctx->initCheck); LZ4_resetStream_fast((LZ4_stream_t *)ctx); - /* Point to the dictionary context */ - internal_ctx->dictCtx = cdict ? &(cdict->fastCtx->internal_donotuse) : NULL; + LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL); } else { if (cdict) { memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx)); -- cgit v0.12 From 3a0c571272731166774405e1f96b3161827ca09f Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 11 Apr 2018 16:31:52 -0400 Subject: Add a LZ4_STATIC_LINKING_ONLY Macro to Guard Experimental APIs --- lib/lz4.c | 1 + lib/lz4.h | 2 ++ lib/lz4frame.c | 1 + 3 files changed, 4 insertions(+) diff --git a/lib/lz4.c b/lib/lz4.c index cee85ec..9e6b1e2 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -89,6 +89,7 @@ /*-************************************ * Dependency **************************************/ +#define LZ4_STATIC_LINKING_ONLY #include "lz4.h" /* see also "memory routines" below */ diff --git a/lib/lz4.h b/lib/lz4.h index f90120e..0dfa19e 100644 --- a/lib/lz4.h +++ b/lib/lz4.h @@ -353,6 +353,7 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or * Their signatures may change. **************************************/ +#ifdef LZ4_STATIC_LINKING_ONLY /*! LZ4_resetStream_fast() : * When an LZ4_stream_t is known to be in a internally coherent state, @@ -414,6 +415,7 @@ LZ4LIB_API int LZ4_compress_fast_extState_fastReset (void* state, const char* sr */ LZ4LIB_API void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream); +#endif /*-************************************ * Private definitions diff --git a/lib/lz4frame.c b/lib/lz4frame.c index d973b5f..7608d90 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -74,6 +74,7 @@ You can contact the author at : * Includes **************************************/ #include "lz4frame_static.h" +#define LZ4_STATIC_LINKING_ONLY #include "lz4.h" #define LZ4_HC_STATIC_LINKING_ONLY #include "lz4hc.h" -- cgit v0.12 From 51a56c47c0d34b6255bdd8e7b33f0ccb44b80351 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 11 Apr 2018 16:55:12 -0400 Subject: Minor Fixes --- lib/lz4.c | 22 +++++++++++++--------- lib/lz4frame.c | 2 -- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index 9e6b1e2..111085a 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -1193,7 +1193,19 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) } void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream) { - working_stream->internal_donotuse.dictCtx = dictionary_stream != NULL ? &(dictionary_stream->internal_donotuse) : NULL; + if (dictionary_stream != NULL) { + /* If the current offset is zero, we will never look in the + * external dictionary context, since there is no value a table + * entry can take that indicate a miss. In that case, we need + * to bump the offset to something non-zero. + */ + if (working_stream->internal_donotuse.currentOffset == 0) { + working_stream->internal_donotuse.currentOffset = 64 KB; + } + working_stream->internal_donotuse.dictCtx = &(dictionary_stream->internal_donotuse); + } else { + working_stream->internal_donotuse.dictCtx = NULL; + } } @@ -1264,14 +1276,6 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t)); result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); } else { - /* If the current offset is zero, we will never look in the - * external dictionary context, since there is no value a table - * entry can take that indicate a miss. In that case, we need - * to bump the offset to something non-zero. - */ - if (streamPtr->currentOffset == 0) { - streamPtr->currentOffset = 64 KB; - } result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration); } } else { diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 7608d90..507e4fe 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -531,8 +531,6 @@ static void LZ4F_applyCDict(void* ctx, const LZ4F_CDict* cdict, int level) { if (level < LZ4HC_CLEVEL_MIN) { - LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t *)ctx)->internal_donotuse; - assert(!internal_ctx->initCheck); LZ4_resetStream_fast((LZ4_stream_t *)ctx); LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL); } else { -- cgit v0.12 From 056ea63215800c9c16e81db92eacc9a9c87548ae Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 11 Apr 2018 18:42:09 -0400 Subject: Fix Silly Warning (const-ness in declaration has no effect on value types!) --- lib/lz4.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/lz4.c b/lib/lz4.c index 111085a..4b0efb1 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -1147,7 +1147,7 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream) MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); } -void LZ4_resetStream_fast(LZ4_stream_t* const ctx) { +void LZ4_resetStream_fast(LZ4_stream_t* ctx) { LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32); } -- cgit v0.12 From 7b3cd10579fdc3d62bbf5e00f07ca19068a4403b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 11 Apr 2018 16:31:43 -0700 Subject: reduced test time on circle-ci --- Makefile | 4 ++-- circle.yml | 14 +++++++------- tests/Makefile | 13 ++++++------- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 3a95808..86613fd 100644 --- a/Makefile +++ b/Makefile @@ -143,10 +143,10 @@ clangtest-native: clean @CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) -C $(TESTDIR) native CC=clang usan: clean - CC=clang CFLAGS="-O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T1mn" NB_LOOPS=-i1 + CC=clang CFLAGS="-O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T30s" NB_LOOPS=-i1 usan32: clean - CFLAGS="-m32 -O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T1mn" NB_LOOPS=-i1 + CFLAGS="-m32 -O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T30s" NB_LOOPS=-i1 staticAnalyze: clean CFLAGS=-g scan-build --status-bugs -v $(MAKE) all diff --git a/circle.yml b/circle.yml index 9a0d1ec..aeb52e9 100644 --- a/circle.yml +++ b/circle.yml @@ -11,10 +11,10 @@ test: - clang -v; make clangtest && make clean - g++ -v; make gpptest && make clean - gcc -v; make c_standards && make clean - - gcc-5 -v; make -C tests test-lz4 CC=gcc-5 MOREFLAGS=-Werror && make clean - - gcc-5 -v; make -C tests test-lz4c32 CC=gcc-5 MOREFLAGS="-I/usr/include/x86_64-linux-gnu -Werror" && make clean - - gcc-6 -v; make c_standards CC=gcc-6 && make clean - - gcc-6 -v; make -C tests test-lz4 CC=gcc-6 MOREFLAGS=-Werror && make clean + - gcc-5 -v; CC=gcc-5 CFLAGS="-O2 -Werror" make check && make clean + - gcc-5 -v; CC=gcc-5 CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean + - gcc-6 -v; CC=gcc-6 make c_standards && make clean + - gcc-6 -v; CC=gcc-6 MOREFLAGS="-O2 -Werror" make check && make clean # Shorter tests - make cmake && make clean - make -C tests test-lz4 @@ -22,11 +22,11 @@ test: - make -C tests test-frametest - make -C tests test-fullbench - make -C tests test-fuzzer && make clean - - make -C lib all && make clean - - pyenv global 3.4.4; CFLAGS=-I/usr/include/x86_64-linux-gnu make versionsTest && make clean + - make -C lib all && make clean + - pyenv global 3.4.4; CPPFLAGS=-I/usr/include/x86_64-linux-gnu make versionsTest && make clean - make travis-install && make clean # Longer tests - - gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean + - gcc -v; CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean - make usan && make clean - clang -v; make staticAnalyze && make clean # Valgrind tests diff --git a/tests/Makefile b/tests/Makefile index 77f5d02..d11b7fe 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -56,7 +56,7 @@ LZ4 := $(PRGDIR)/lz4$(EXT) # Default test parameters TEST_FILES := COPYING -FUZZER_TIME := -T3mn +FUZZER_TIME := -T90s NB_LOOPS ?= -i1 @@ -393,13 +393,12 @@ test-mem: lz4 datagen fuzzer frametest fullbench ./datagen -g16KB -s2 > ftmdg16K2 ./datagen -g16KB -s3 > ftmdg16K3 valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --force --multiple ftmdg16K ftmdg16K2 ftmdg16K3 - ./datagen -g16MB > ftmdg16M - valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f ftmdg16M ftmdg16K2 + ./datagen -g7MB > ftmdg7M + valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f ftmdg7M ftmdg16K2 valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -t ftmdg16K2 - valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 ftmdg16M - valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 ftmdg16M ftmdg16K2 - ./datagen -g256MB > ftmdg256M - valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq ftmdg256M $(VOID) + valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 ftmdg7M + valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 ftmdg7M ftmdg16K2 + valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq ftmdg7M $(VOID) $(RM) ftm* valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i64 -t1 valgrind --leak-check=yes --error-exitcode=1 ./frametest -i256 -- cgit v0.12 From ca388790e689ddb09e5e5f7f86a5bdb17dee67d7 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 11 Apr 2018 16:41:25 -0700 Subject: allow system-defined CPPFLAGS in /tests --- tests/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index d11b7fe..25f00b8 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -39,7 +39,7 @@ CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \ -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \ -Wpointer-arith -Wstrict-aliasing=1 CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) -CPPFLAGS:= -I$(LZ4DIR) -I$(PRGDIR) -DXXH_NAMESPACE=LZ4_ +CPPFLAGS+= -I$(LZ4DIR) -I$(PRGDIR) -DXXH_NAMESPACE=LZ4_ FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) -- cgit v0.12 From 1838803948ceaf211b9e79fb405ef9b7340762ce Mon Sep 17 00:00:00 2001 From: test4973 Date: Wed, 11 Apr 2018 16:49:40 -0700 Subject: fixed LZ4_compress_fast_extState_fastReset() --- lib/lz4.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index d79b2cc..d564ddc 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -953,7 +953,7 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int * (see comment in lz4.h on LZ4_resetStream_fast() for a definition of * "correctly initialized"). */ -int LZ4_compress_fast_extState_fastReset(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration) { LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse; if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; @@ -961,8 +961,7 @@ int LZ4_compress_fast_extState_fastReset(void* state, const char* source, char* if (dstCapacity >= LZ4_compressBound(srcSize)) { if (srcSize < LZ4_64Klimit) { const tableType_t tableType = byU16; - LZ4_prepareTable(ctx, inputSize, tableType); -LZ4_prepareTable>>> dev + LZ4_prepareTable(ctx, srcSize, tableType); if (ctx->currentOffset) { return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, dictSmall, acceleration); } else { @@ -970,13 +969,13 @@ LZ4_prepareTable>>> dev } } else { const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr; - LZ4_prepareTable(ctx, inputSize, tableType); - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + LZ4_prepareTable(ctx, srcSize, tableType); + return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration); } } else { if (srcSize < LZ4_64Klimit) { const tableType_t tableType = byU16; - LZ4_prepareTable(ctx, inputSize, tableType); + LZ4_prepareTable(ctx, srcSize, tableType); if (ctx->currentOffset) { return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration); } else { @@ -984,8 +983,8 @@ LZ4_prepareTable>>> dev } } else { const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr; - LZ4_prepareTable(ctx, inputSize, tableType); - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration); + LZ4_prepareTable(ctx, srcSize, tableType); + return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); } } } -- cgit v0.12 From d6d9bccd537e307cc0bf3f8475801dea79e97367 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 12 Apr 2018 06:47:27 -0700 Subject: modified versionsTest to use MOREFLAGS rather CPPFLAGS as some older versions of LZ4 overwrite CPPFLAGS environment variable. --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index aeb52e9..fa37590 100644 --- a/circle.yml +++ b/circle.yml @@ -23,7 +23,7 @@ test: - make -C tests test-fullbench - make -C tests test-fuzzer && make clean - make -C lib all && make clean - - pyenv global 3.4.4; CPPFLAGS=-I/usr/include/x86_64-linux-gnu make versionsTest && make clean + - pyenv global 3.4.4; make versionsTest MOREFLAGS=-I/usr/include/x86_64-linux-gnu && make clean - make travis-install && make clean # Longer tests - gcc -v; CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean -- cgit v0.12 From 8af32ce6f7109ecf5cd7d73527e0aba3a63b55e5 Mon Sep 17 00:00:00 2001 From: test4973 Date: Thu, 12 Apr 2018 07:25:40 -0700 Subject: modified a few traces for debug --- lib/lz4.c | 5 +++-- lib/lz4hc.c | 7 +++---- tests/fuzzer.c | 4 +++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index d564ddc..e4a68cd 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -594,7 +594,6 @@ LZ4_FORCE_INLINE void LZ4_prepareTable( cctx->tableType = clearedTable; } else { DEBUGLOG(4, "Re-use hash table (no reset)"); - //if (tableType == byU32) cctx->currentOffset += 64 KB; } } @@ -602,9 +601,11 @@ LZ4_FORCE_INLINE void LZ4_prepareTable( * than compressing without a gap. However, compressing with * currentOffset == 0 is faster still, so we preserve that case. */ + DEBUGLOG(2, "tableType=%u, currentOffset=%u", cctx->tableType, cctx->currentOffset); if (cctx->currentOffset != 0 && tableType == byU32) { cctx->currentOffset += 64 KB; } + DEBUGLOG(2, "currentOffset: %u", cctx->currentOffset); /* Finally, clear history */ cctx->dictCtx = NULL; @@ -1267,7 +1268,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) const BYTE* const dictEnd = p + dictSize; const BYTE* base; - DEBUGLOG(4, "LZ4_loadDict %p", LZ4_dict); + DEBUGLOG(4, "LZ4_loadDict (%p into %p)", dictionary, LZ4_dict); LZ4_prepareTable(dict, 0, tableType); diff --git a/lib/lz4hc.c b/lib/lz4hc.c index d05760b..111a09b 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -336,7 +336,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( U32 const mlAdd = (matchLength>=19) ? ((matchLength-19) / 255) + 1 : 0; U32 const cost = 1 + llAdd + ll + 2 + mlAdd; if (start==NULL) start = *anchor; /* only works for single segment */ - //g_debuglog_enable = (pos >= 2228) & (pos <= 2262); + /* g_debuglog_enable = (pos >= 2228) & (pos <= 2262); */ DEBUGLOG(6, "pos:%7u -- literals:%3u, match:%4i, offset:%5u, cost:%3u + %u", pos, (U32)(*ip - *anchor), matchLength, (U32)(*ip-match), @@ -1137,15 +1137,14 @@ static int LZ4HC_compress_optimal ( encode: /* cur, last_match_pos, best_mlen, best_off must be set */ assert(cur < LZ4_OPT_NUM); assert(last_match_pos >= 1); /* == 1 when only one candidate */ - DEBUGLOG(6, "reverse traversal, looking for shortest path") - DEBUGLOG(6, "last_match_pos = %i", last_match_pos); + DEBUGLOG(6, "reverse traversal, looking for shortest path (last_match_pos=%i)", last_match_pos); { int candidate_pos = cur; int selected_matchLength = best_mlen; int selected_offset = best_off; while (1) { /* from end to beginning */ int const next_matchLength = opt[candidate_pos].mlen; /* can be 1, means literal */ int const next_offset = opt[candidate_pos].off; - DEBUGLOG(6, "pos %i: sequence length %i", candidate_pos, selected_matchLength); + DEBUGLOG(7, "pos %i: sequence length %i", candidate_pos, selected_matchLength); opt[candidate_pos].mlen = selected_matchLength; opt[candidate_pos].off = selected_offset; selected_matchLength = next_matchLength; diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 2e3ee92..a650277 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -632,13 +632,15 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize); FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary loaded with LZ4_loadDict()"); + DISPLAYLEVEL(5, " compress %i bytes from buffer(%p) into dst(%p) using dict(%p) of size %i \n", blockSize, block, decodedBuffer, dict, dictSize); LZ4_loadDict(&LZ4dict, dict, dictSize); ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue should work : enough size available within output buffer"); /* Decompress with dictionary as external */ - FUZ_DISPLAYTEST("test LZ4_decompress_fast_usingDict(): decoding %i bytes, dict(%p) of size %i", blockSize, dict, dictSize); + FUZ_DISPLAYTEST("test LZ4_decompress_fast_usingDict() with dictionary as extDict"); + DISPLAYLEVEL(5, " decoding %i bytes from buffer(%p) using dict(%p) of size %i \n", blockSize, decodedBuffer, dict, dictSize); decodedBuffer[blockSize] = 0; ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input"); -- cgit v0.12 From db9aa785c51bbcae50c777e89fb537393bfca856 Mon Sep 17 00:00:00 2001 From: test4973 Date: Thu, 12 Apr 2018 16:12:21 -0700 Subject: fixed : counting matches which overlap extDict and prefix --- lib/lz4.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index e4a68cd..21892d3 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -601,11 +601,9 @@ LZ4_FORCE_INLINE void LZ4_prepareTable( * than compressing without a gap. However, compressing with * currentOffset == 0 is faster still, so we preserve that case. */ - DEBUGLOG(2, "tableType=%u, currentOffset=%u", cctx->tableType, cctx->currentOffset); if (cctx->currentOffset != 0 && tableType == byU32) { cctx->currentOffset += 64 KB; } - DEBUGLOG(2, "currentOffset: %u", cctx->currentOffset); /* Finally, clear history */ cctx->dictCtx = NULL; @@ -652,7 +650,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( const BYTE* dictBase = dictDirective == usingDictCtx ? dictionary + dictSize - dictCtx->currentOffset : /* is it possible that dictCtx->currentOffset != dictCtx->dictSize ? */ dictionary + dictSize - startIndex; - const BYTE* dictLowLimit; BYTE* op = (BYTE*) dest; BYTE* const olimit = op + maxOutputSize; @@ -665,7 +662,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */ lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0); - dictLowLimit = dictionary ? dictionary : lowLimit; if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ @@ -735,15 +731,16 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( /* there was no match, try the dictionary */ matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); match = dictBase + matchIndex; - lowLimit = dictLowLimit; + lowLimit = dictionary; } else { match = base + matchIndex; lowLimit = (const BYTE*)source; } } else if (dictDirective==usingExtDict) { if (matchIndex < startIndex) { + DEBUGLOG(7, "extDict candidate: matchIndex=%5u < startIndex=%5u", matchIndex, startIndex); match = dictBase + matchIndex; - lowLimit = dictLowLimit; + lowLimit = dictionary; } else { match = base + matchIndex; lowLimit = (const BYTE*)source; @@ -786,6 +783,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( /* Copy Literals */ LZ4_wildCopy(op, anchor, op+litLength); op+=litLength; + DEBUGLOG(6, "seq.start:%zi, literals=%u, match.start:%zi", anchor-(const BYTE*)source, litLength, ip-(const BYTE*)source); } _next_match: @@ -793,29 +791,33 @@ _next_match: if (maybe_ext_memSegment) { /* static test */ assert(offset <= MAX_DISTANCE && offset > 0); LZ4_writeLE16(op, (U16)offset); op+=2; + DEBUGLOG(6, " with offset=%u (ext if > %zi)", offset, ip - (const BYTE*)source); } else { assert(ip-match <= MAX_DISTANCE); LZ4_writeLE16(op, (U16)(ip - match)); op+=2; + DEBUGLOG(6, " with offset=%u (same segment)", (U32)(ip - match)); } /* Encode MatchLength */ { unsigned matchCode; if ( (dictDirective==usingExtDict || dictDirective==usingDictCtx) - && (lowLimit==dictionary) ) { - const BYTE* limit; - limit = ip + (dictEnd-match); + && (lowLimit==dictionary) /* match within extDict */ ) { + const BYTE* limit = ip + (dictEnd-match); + assert(dictEnd > match); if (limit > matchlimit) limit = matchlimit; matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); ip += MINMATCH + matchCode; if (ip==limit) { - unsigned const more = LZ4_count(ip, (const BYTE*)source, matchlimit); + unsigned const more = LZ4_count(limit, (const BYTE*)source, matchlimit); matchCode += more; ip += more; } + DEBUGLOG(6, " with matchLength=%u starting in extDict", matchCode+MINMATCH); } else { matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); ip += MINMATCH + matchCode; + DEBUGLOG(6, " with matchLength=%u", matchCode+MINMATCH); } if ( outputLimited && /* Check output buffer overflow */ @@ -865,14 +867,18 @@ _next_match: /* there was no match, try the dictionary */ matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); match = dictBase + matchIndex; + lowLimit = dictionary; /* required for match length counter */ } else { match = base + matchIndex; + lowLimit = (const BYTE*)source; /* required for match length counter */ } } else if (dictDirective==usingExtDict) { if (matchIndex < startIndex) { match = dictBase + matchIndex; + lowLimit = dictionary; /* required for match length counter */ } else { match = base + matchIndex; + lowLimit = (const BYTE*)source; /* required for match length counter */ } } else { /* single memory segment */ match = base + matchIndex; @@ -885,6 +891,7 @@ _next_match: *token=0; if (maybe_ext_memSegment) offset = current - matchIndex; + DEBUGLOG(6, "seq.start:%zi, literals=%u, match.start:%zi", anchor-(const BYTE*)source, 0, ip-(const BYTE*)source); goto _next_match; } } -- cgit v0.12 From 98811d606808da9f59affd990670ba34e5a9bee2 Mon Sep 17 00:00:00 2001 From: Cyan4973 Date: Fri, 13 Apr 2018 00:59:27 -0700 Subject: added sudo rights for low-mem-address tests --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a446420..0a876f9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ matrix: env: Ubu=12.04cont Cmd='make -C tests test-lz4 test-lz4c test-fullbench' COMPILER=cc - os: linux - sudo: false + sudo: required env: Ubu=12.04cont Cmd='sudo sysctl -w vm.mmap_min_addr="4096" && make -C tests test-frametest test-fuzzer' COMPILER=cc - os: linux -- cgit v0.12 From 57afa36795f478d0f9b069ad19b578761e3fb16a Mon Sep 17 00:00:00 2001 From: Cyan4973 Date: Fri, 13 Apr 2018 01:01:54 -0700 Subject: compatibility with gcc-4.4 string.h version Someone found it would be a great idea to define there a global variable under the very generic name "index". Cause problem with shadow warnings, so no variable can be named "index" now ... Also : automatically update API manual --- doc/lz4_manual.html | 94 ++++++++++++++++++++++++++++++++++++++++------------- lib/lz4.c | 6 ++-- 2 files changed, 75 insertions(+), 25 deletions(-) diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html index d14467f..f8639fe 100644 --- a/doc/lz4_manual.html +++ b/doc/lz4_manual.html @@ -15,8 +15,9 @@
  • Advanced Functions
  • Streaming Compression Functions
  • Streaming Decompression Functions
    • -
  • Private definitions
    • -
  • Obsolete Functions
    • +
  • Unstable declarations
    • +
  • Private definitions
    • +
  • Obsolete Functions

    • Introduction

    @@ -245,22 +246,80 @@ int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize,

    -

    Private definitions

    +
    Unstable declarations
    + Declarations in this section should be considered unstable.
+ Use at your own peril, etc., etc.
+ They may be removed in the future.
+ Their signatures may change.
+
    
+
+
    void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
+
      When an LZ4_stream_t is known to be in a internally coherent state,
+  it can often be prepared for a new compression with almost no work, only
+  sometimes falling back to the full, expensive reset that is always required
+  when the stream is in an indeterminate state (i.e., the reset performed by
+  LZ4_resetStream()).
+
+  LZ4_streams are guaranteed to be in a valid state when:
+  - returned from LZ4_createStream()
+  - reset by LZ4_resetStream()
+  - memset(stream, 0, sizeof(LZ4_stream_t))
+  - the stream was in a valid state and was reset by LZ4_resetStream_fast()
+  - the stream was in a valid state and was then used in any compression call
+    that returned success
+  - the stream was in an indeterminate state and was used in a compression
+    call that fully reset the state (LZ4_compress_fast_extState()) and that
+    returned success
+ 
+

    
+
+
    int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
+
      A variant of LZ4_compress_fast_extState().
+
+  Using this variant avoids an expensive initialization step. It is only safe
+  to call if the state buffer is known to be correctly initialized already
+  (see above comment on LZ4_resetStream_fast() for a definition of "correctly
+  initialized"). From a high level, the difference is that this function
+  initializes the provided state with a call to LZ4_resetStream_fast() while
+  LZ4_compress_fast_extState() starts with a call to LZ4_resetStream().
+ 
+

    
+
+
    void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream);
+
      This is an experimental API that allows for the efficient use of a
+  static dictionary many times.
+
+  Rather than re-loading the dictionary buffer into a working context before
+  each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a
+  working LZ4_stream_t, this function introduces a no-copy setup mechanism,
+  in which the working stream references the dictionary stream in-place.
+
+  Several assumptions are made about the state of the dictionary stream.
+  Currently, only streams which have been prepared by LZ4_loadDict() should
+  be expected to work.
+
+  Alternatively, the provided dictionary stream pointer may be NULL, in which
+  case any existing dictionary stream is unset.
+
+  If a dictionary is provided, it replaces any pre-existing stream history.
+  The dictionary contents are the only history that can be referenced and
+  logically immediately precede the data compressed in the first subsequent
+  compression call.
+
+  The dictionary will only remain attached to the working stream through the
+  first compression call, at the end of which it is cleared. The dictionary
+  stream (and source buffer) must remain in-place / accessible / unchanged
+  through the completion of the first compression call on the stream.
+ 
+

    
+
+
    Private definitions
      Do not use these definitions.
  They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`.
  Using these definitions will expose code to API and/or ABI break in future versions of the library.
 
    
 
 
    typedef struct {
-    uint32_t hashTable[LZ4_HASH_SIZE_U32];
-    uint32_t currentOffset;
-    uint32_t initCheck;
-    const uint8_t* dictionary;
-    uint8_t* bufferStart;   /* obsolete, used for slideInputBuffer */
-    uint32_t dictSize;
-} LZ4_stream_t_internal;
-

    
-
    typedef struct {
     const uint8_t* externalDict;
     size_t extDictSize;
     const uint8_t* prefixEnd;
@@ -268,15 +327,6 @@ int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize,
 } LZ4_streamDecode_t_internal;
 

    
 
    typedef struct {
-    unsigned int hashTable[LZ4_HASH_SIZE_U32];
-    unsigned int currentOffset;
-    unsigned int initCheck;
-    const unsigned char* dictionary;
-    unsigned char* bufferStart;   /* obsolete, used for slideInputBuffer */
-    unsigned int dictSize;
-} LZ4_stream_t_internal;
-

    
-
    typedef struct {
     const unsigned char* externalDict;
     size_t extDictSize;
     const unsigned char* prefixEnd;
@@ -311,7 +361,7 @@ union LZ4_streamDecode_u {
  
 

    
 
-
    Obsolete Functions
    
+
    Obsolete Functions
    
 
 
    #ifdef LZ4_DISABLE_DEPRECATE_WARNINGS
 #  define LZ4_DEPRECATED(message)   /* disable deprecation warnings */
diff --git a/lib/lz4.c b/lib/lz4.c
index 21892d3..f55e4e1 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -517,15 +517,15 @@ LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tab
     return LZ4_hash4(LZ4_read32(p), tableType);
 }
 
-static void LZ4_putIndexOnHash(U32 index, U32 h, void* tableBase, tableType_t const tableType)
+static void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t const tableType)
 {
     switch (tableType)
     {
     default: /* fallthrough */
     case clearedTable: /* fallthrough */
     case byPtr: { /* illegal! */ assert(0); return; }
-    case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = index; return; }
-    case byU16: { U16* hashTable = (U16*) tableBase; assert(index < 65536); hashTable[h] = (U16)index; return; }
+    case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = idx; return; }
+    case byU16: { U16* hashTable = (U16*) tableBase; assert(idx < 65536); hashTable[h] = (U16)idx; return; }
     }
 }
 
-- 
cgit v0.12


From 54ec83ce1f62014398b76a441d1ff1212dad9604 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 13 Apr 2018 02:10:53 -0700
Subject: fixed potential ptrdiff_t overflow (32-bits mode)

Also removed pointer comparison, which should solve #485
---
 lib/lz4.c | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index f55e4e1..c48aca4 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1315,15 +1315,14 @@ void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dic
 }
 
 
-static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src)
+static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, int nextSize)
 {
-    if ((LZ4_dict->currentOffset > 0x80000000) ||
-        ((uptrval)LZ4_dict->currentOffset > (uptrval)src)) {   /* address space overflow */
+    if (LZ4_dict->currentOffset + nextSize > 0x80000000) {   /* potential ptrdiff_t overflow (32-bits mode) */
         /* rescale hash table */
         U32 const delta = LZ4_dict->currentOffset - 64 KB;
         const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize;
         int i;
-        DEBUGLOG(4, "LZ4_renormDictT %p", LZ4_dict);
+        DEBUGLOG(4, "LZ4_renormDictT");
         for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0;
             else LZ4_dict->hashTable[i] -= delta;
@@ -1341,10 +1340,8 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
     LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse;
     const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
 
-    const BYTE* smallest = (const BYTE*) source;
     if (streamPtr->initCheck) return 0;   /* Uninitialized structure detected */
-    if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd;
-    LZ4_renormDictT(streamPtr, smallest);
+    LZ4_renormDictT(streamPtr, inputSize);   /* avoid index overflow */
     if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
 
     /* Check overlapping input/dictionary space */
@@ -1377,7 +1374,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
             if (inputSize > 4 KB) {
                 /* For compressing large blobs, it is faster to pay the setup
                  * cost to copy the dictionary's tables into the active context,
-                 * so that the compression loop is only looking in one table.
+                 * so that the compression loop is only looking into one table.
                  */
                 memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
@@ -1398,8 +1395,8 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
 }
 
 
-/* Hidden debug function, to force external dictionary mode */
-int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize)
+/* Hidden debug function, to force-test external dictionary mode */
+int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize)
 {
     LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse;
     int result;
@@ -1407,16 +1404,16 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
 
     const BYTE* smallest = dictEnd;
     if (smallest > (const BYTE*) source) smallest = (const BYTE*) source;
-    LZ4_renormDictT(streamPtr, smallest);
+    LZ4_renormDictT(streamPtr, srcSize);
 
     if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
-        result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, dictSmall, 1);
+        result = LZ4_compress_generic(streamPtr, source, dest, srcSize, 0, notLimited, byU32, usingExtDict, dictSmall, 1);
     } else {
-        result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
+        result = LZ4_compress_generic(streamPtr, source, dest, srcSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
     }
 
     streamPtr->dictionary = (const BYTE*)source;
-    streamPtr->dictSize = (U32)inputSize;
+    streamPtr->dictSize = (U32)srcSize;
 
     return result;
 }
-- 
cgit v0.12


From c40bac31d3886c2f69d0364823b6d6aaa972ee5b Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 13 Apr 2018 02:26:14 -0700
Subject: added comment on variables required after _next_match

---
 lib/lz4.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/lib/lz4.c b/lib/lz4.c
index c48aca4..54d037c 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -787,6 +787,14 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         }
 
 _next_match:
+        /* at this stage, the following variables must be correctly set :
+         * - ip : at start of LZ operation
+         * - match : at start of previous pattern occurence; can be within current prefix, or within extDict
+         * - offset : if maybe_ext_memSegment==1 (constant)
+         * - lowLimit : must be == dictionary to mean "match is within extDict"; must be == source otherwise
+         * - token and *token : position to write 4-bits for match length; higher 4-bits for literal length supposed already written
+         */
+
         /* Encode Offset */
         if (maybe_ext_memSegment) {   /* static test */
             assert(offset <= MAX_DISTANCE && offset > 0);
-- 
cgit v0.12


From d2bcfa31f525aaa11c2d248af0ba487791399c1f Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 13 Apr 2018 02:45:32 -0700
Subject: fixed minor unused variable warning

---
 lib/lz4.c | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 54d037c..e8f9831 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1408,10 +1408,7 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
 {
     LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse;
     int result;
-    const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
 
-    const BYTE* smallest = dictEnd;
-    if (smallest > (const BYTE*) source) smallest = (const BYTE*) source;
     LZ4_renormDictT(streamPtr, srcSize);
 
     if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
-- 
cgit v0.12


From 6dd64e0776dccf5c19e8438625657abf0e6f3048 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Thu, 12 Apr 2018 18:23:01 -0400
Subject: Add Tests for LZ4_attach_dictionary and Friends

---
 tests/fuzzer.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 78 insertions(+)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index c134fe3..98b3d9f 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -43,6 +43,7 @@
 #include       /* fgets, sscanf */
 #include      /* strcmp */
 #include        /* clock_t, clock, CLOCKS_PER_SEC */
+#define LZ4_STATIC_LINKING_ONLY
 #define LZ4_HC_STATIC_LINKING_ONLY
 #include "lz4hc.h"
 #define XXH_STATIC_LINKING_ONLY
@@ -402,6 +403,11 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
         FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed");
 
+        /* Test compression using fast reset external state*/
+        FUZ_DISPLAYTEST;
+        ret = LZ4_compress_fast_extState_fastReset(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
+        FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState_fastReset() failed");
+
         /* Test compression */
         FUZ_DISPLAYTEST;
         ret = LZ4_compress_default(block, compressedBuffer, blockSize, (int)compressedBufferSize);
@@ -626,6 +632,78 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
                 FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize);
         }   }
 
+        /* Compress using external dictionary stream */
+        FUZ_DISPLAYTEST;
+        {
+            LZ4_stream_t LZ4_stream;
+
+            LZ4_loadDict(&LZ4dict, dict, dictSize);
+
+            LZ4_resetStream(&LZ4_stream);
+            LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
+            blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
+            FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue using extDictCtx failed");
+
+            FUZ_DISPLAYTEST;
+            LZ4_resetStream(&LZ4_stream);
+            LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
+            ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
+            FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using extDictCtx should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
+
+            FUZ_DISPLAYTEST;
+            LZ4_resetStream(&LZ4_stream);
+            LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
+            ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
+            FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
+            FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx should work : enough size available within output buffer");
+
+            FUZ_DISPLAYTEST;
+            LZ4_resetStream_fast(&LZ4_stream);
+            LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
+            ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
+            FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
+            FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx with re-used context should work : enough size available within output buffer");
+        }
+
+        /* Decompress with dictionary as external */
+        FUZ_DISPLAYTEST;
+        decodedBuffer[blockSize] = 0;
+        ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
+        FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
+        FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
+        crcCheck = XXH32(decodedBuffer, blockSize, 0);
+        if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
+        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
+
+        FUZ_DISPLAYTEST;
+        decodedBuffer[blockSize] = 0;
+        ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
+        FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
+        FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
+        crcCheck = XXH32(decodedBuffer, blockSize, 0);
+        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+
+        FUZ_DISPLAYTEST;
+        decodedBuffer[blockSize-1] = 0;
+        ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
+        FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
+        FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
+
+        FUZ_DISPLAYTEST;
+        decodedBuffer[blockSize-1] = 0;
+        ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
+        FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
+        FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
+
+        FUZ_DISPLAYTEST;
+        {   U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
+            if ((U32)blockSize > missingBytes) {
+                decodedBuffer[blockSize-missingBytes] = 0;
+                ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-missingBytes, dict, dictSize);
+                FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%u byte)", missingBytes);
+                FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize);
+        }   }
+
         /* Compress HC using External dictionary */
         FUZ_DISPLAYTEST;
         dict -= (FUZ_rand(&randState) & 7);    /* even bigger separation */
-- 
cgit v0.12


From 9f0f6b89bb27d4d6c5d12a7142fc330f3d1b3a10 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Thu, 12 Apr 2018 19:17:53 -0400
Subject: Further Test that ExtDictCtx Mode Produces the Exact Same Output

---
 tests/fuzzer.c | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 98b3d9f..db05738 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -303,9 +303,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         U32 testNb = 0;
         U32 randState = FUZ_rand(&coreRandState) ^ PRIME3;
         int const blockSize  = (FUZ_rand(&randState) % (FUZ_MAX_BLOCK_SIZE-1)) + 1;
-        int const blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize);
+        int const blockStart = (FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize - 1)) + 1;
         int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
-        int const dictSize = MIN(dictSizeRand, blockStart);
+        int const dictSize = MIN(dictSizeRand, blockStart - 1);
         int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1);
         char* const block = ((char*)CNBuffer) + blockStart;
         const char* dict = block - dictSize;
@@ -636,6 +636,13 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_DISPLAYTEST;
         {
             LZ4_stream_t LZ4_stream;
+            int expectedSize;
+            U32 expectedCrc;
+
+            LZ4_loadDict(&LZ4dict, dict, dictSize);
+            expectedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
+            FUZ_CHECKTEST(expectedSize<=0, "LZ4_compress_fast_continue reference compression for extDictCtx should have succeeded");
+            expectedCrc = XXH32(compressedBuffer, expectedSize, 0);
 
             LZ4_loadDict(&LZ4dict, dict, dictSize);
 
@@ -644,6 +651,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
             FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue using extDictCtx failed");
 
+            /* In the future, it might be desirable to let extDictCtx mode's
+             * output diverge from the output generated by regular extDict mode.
+             * Until that time, this comparison serves as a good regression
+             * test.
+             */
+            FUZ_CHECKTEST(blockContinueCompressedSize != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output (%d expected vs %d actual)", expectedSize, blockContinueCompressedSize);
+            FUZ_CHECKTEST(XXH32(compressedBuffer, blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
+
             FUZ_DISPLAYTEST;
             LZ4_resetStream(&LZ4_stream);
             LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
@@ -656,6 +671,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
             FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
             FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx should work : enough size available within output buffer");
+            FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output");
+            FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
 
             FUZ_DISPLAYTEST;
             LZ4_resetStream_fast(&LZ4_stream);
@@ -663,6 +680,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
             FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
             FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx with re-used context should work : enough size available within output buffer");
+            FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output");
+            FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
         }
 
         /* Decompress with dictionary as external */
-- 
cgit v0.12


From e9280647976ca468ab67ffbf3dd9c8e533fe23bf Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 16 Apr 2018 15:11:28 -0700
Subject: fixed gcc performance regression

---
 lib/lz4.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index e8f9831..c2012f6 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -660,6 +660,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     /* Init conditions */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;   /* Unsupported inputSize, too large (or negative) */
     if (tableType==byPtr) assert(dictDirective==noDict);      /* only supported use case with byPtr */
+    assert(acceleration >= 1);
 
     lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0);
 
@@ -712,6 +713,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         } else {   /* byU32, byU16 */
 
             const BYTE* forwardIp = ip;
+            unsigned step = 1;
             unsigned searchMatchNb = acceleration << LZ4_skipTrigger;
             do {
                 U32 const h = forwardH;
@@ -720,8 +722,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 assert(matchIndex <= current);
                 assert(forwardIp - base < (ptrdiff_t)(2 GB - 1));
                 ip = forwardIp;
-                assert(searchMatchNb >= (1<> LZ4_skipTrigger);
+                forwardIp += step;
+                step = (searchMatchNb++ >> LZ4_skipTrigger);
 
                 if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals;
                 assert(ip < mflimitPlusOne);
-- 
cgit v0.12


From 4aff9b10b56e44b2ff6326f611c3659565812d80 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 16 Apr 2018 16:14:28 -0700
Subject: fixed fuzzer tests

which were modified in parallel within branc `dev`
---
 tests/fuzzer.c | 32 +++++++++++++++++---------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 83ab50b..7721345 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -454,7 +454,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed");
 
         /* Test compression using fast reset external state*/
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         ret = LZ4_compress_fast_extState_fastReset(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
         FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState_fastReset() failed");
 
@@ -687,7 +687,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }   }
 
         /* Compress using external dictionary stream */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         {
             LZ4_stream_t LZ4_stream;
             int expectedSize;
@@ -713,13 +713,13 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_CHECKTEST(blockContinueCompressedSize != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output (%d expected vs %d actual)", expectedSize, blockContinueCompressedSize);
             FUZ_CHECKTEST(XXH32(compressedBuffer, blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
 
-            FUZ_DISPLAYTEST;
+            FUZ_DISPLAYTEST();
             LZ4_resetStream(&LZ4_stream);
             LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
             ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
             FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using extDictCtx should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
 
-            FUZ_DISPLAYTEST;
+            FUZ_DISPLAYTEST();
             LZ4_resetStream(&LZ4_stream);
             LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
             ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
@@ -728,7 +728,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output");
             FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
 
-            FUZ_DISPLAYTEST;
+            FUZ_DISPLAYTEST();
             LZ4_resetStream_fast(&LZ4_stream);
             LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
             ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
@@ -739,36 +739,38 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }
 
         /* Decompress with dictionary as external */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize] = 0;
         ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
         FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
         FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
-        crcCheck = XXH32(decodedBuffer, blockSize, 0);
-        if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
-        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
+        {   U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+            if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
+            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
+        }
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize] = 0;
         ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
         FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
         FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
-        crcCheck = XXH32(decodedBuffer, blockSize, 0);
-        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+        {   U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+        }
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize-1] = 0;
         ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
         FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
         FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize-1] = 0;
         ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
         FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
         FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         {   U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
             if ((U32)blockSize > missingBytes) {
                 decodedBuffer[blockSize-missingBytes] = 0;
-- 
cgit v0.12


From a3aeb34184e20d51616beccfcbbe7aade3cc3a64 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 16 Apr 2018 16:54:03 -0700
Subject: fixed minor format warnings

---
 lib/lz4.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index c2012f6..980a5fd 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -785,7 +785,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
             /* Copy Literals */
             LZ4_wildCopy(op, anchor, op+litLength);
             op+=litLength;
-            DEBUGLOG(6, "seq.start:%zi, literals=%u, match.start:%zi", anchor-(const BYTE*)source, litLength, ip-(const BYTE*)source);
+            DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", (int)(anchor-(const BYTE*)source), litLength, (int)(ip-(const BYTE*)source));
         }
 
 _next_match:
@@ -801,7 +801,7 @@ _next_match:
         if (maybe_ext_memSegment) {   /* static test */
             assert(offset <= MAX_DISTANCE && offset > 0);
             LZ4_writeLE16(op, (U16)offset); op+=2;
-            DEBUGLOG(6, "                with offset=%u  (ext if > %zi)", offset, ip - (const BYTE*)source);
+            DEBUGLOG(6, "                with offset=%u  (ext if > %i)", offset, (int)(ip - (const BYTE*)source));
         } else  {
             assert(ip-match <= MAX_DISTANCE);
             LZ4_writeLE16(op, (U16)(ip - match)); op+=2;
@@ -901,7 +901,7 @@ _next_match:
                 *token=0;
                 if (maybe_ext_memSegment)
                     offset = current - matchIndex;
-                DEBUGLOG(6, "seq.start:%zi, literals=%u, match.start:%zi", anchor-(const BYTE*)source, 0, ip-(const BYTE*)source);
+                DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source));
                 goto _next_match;
             }
         }
-- 
cgit v0.12


From 444211d2599a2be59e3f50418b46ec2431288e9a Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 16 Apr 2018 17:15:02 -0700
Subject: edited a few traces for debugging

---
 lib/lz4.c      | 14 +++++++-------
 tests/fuzzer.c |  6 +++---
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 980a5fd..dca4d69 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -588,12 +588,12 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
           || tableType == byPtr
           || inputSize >= 4 KB)
         {
-            DEBUGLOG(4, "Resetting table in %p", cctx);
+            DEBUGLOG(4, "LZ4_prepareTable: Resetting table in %p", cctx);
             MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
             cctx->currentOffset = 0;
             cctx->tableType = clearedTable;
         } else {
-            DEBUGLOG(4, "Re-use hash table (no reset)");
+            DEBUGLOG(4, "LZ4_prepareTable: Re-use hash table (no reset)");
         }
     }
 
@@ -799,13 +799,13 @@ _next_match:
 
         /* Encode Offset */
         if (maybe_ext_memSegment) {   /* static test */
+            DEBUGLOG(6, "             with offset=%u  (ext if > %i)", offset, (int)(ip - (const BYTE*)source));
             assert(offset <= MAX_DISTANCE && offset > 0);
             LZ4_writeLE16(op, (U16)offset); op+=2;
-            DEBUGLOG(6, "                with offset=%u  (ext if > %i)", offset, (int)(ip - (const BYTE*)source));
         } else  {
+            DEBUGLOG(6, "             with offset=%u  (same segment)", (U32)(ip - match));
             assert(ip-match <= MAX_DISTANCE);
             LZ4_writeLE16(op, (U16)(ip - match)); op+=2;
-            DEBUGLOG(6, "                with offset=%u  (same segment)", (U32)(ip - match));
         }
 
         /* Encode MatchLength */
@@ -823,11 +823,11 @@ _next_match:
                     matchCode += more;
                     ip += more;
                 }
-                DEBUGLOG(6, "                with matchLength=%u starting in extDict", matchCode+MINMATCH);
+                DEBUGLOG(6, "             with matchLength=%u starting in extDict", matchCode+MINMATCH);
             } else {
                 matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit);
                 ip += MINMATCH + matchCode;
-                DEBUGLOG(6, "                with matchLength=%u", matchCode+MINMATCH);
+                DEBUGLOG(6, "             with matchLength=%u", matchCode+MINMATCH);
             }
 
             if ( outputLimited &&    /* Check output buffer overflow */
@@ -1259,7 +1259,7 @@ LZ4_stream_t* LZ4_createStream(void)
 
 void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
 {
-    DEBUGLOG(5, "LZ4_resetStream %p", LZ4_stream);
+    DEBUGLOG(5, "LZ4_resetStream (ctx:%p)", LZ4_stream);
     MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
 }
 
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 7721345..244cc4f 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -687,19 +687,19 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }   }
 
         /* Compress using external dictionary stream */
-        FUZ_DISPLAYTEST();
         {
             LZ4_stream_t LZ4_stream;
             int expectedSize;
             U32 expectedCrc;
 
+            FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_loadDict()");
             LZ4_loadDict(&LZ4dict, dict, dictSize);
             expectedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
             FUZ_CHECKTEST(expectedSize<=0, "LZ4_compress_fast_continue reference compression for extDictCtx should have succeeded");
             expectedCrc = XXH32(compressedBuffer, expectedSize, 0);
 
+            FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary()");
             LZ4_loadDict(&LZ4dict, dict, dictSize);
-
             LZ4_resetStream(&LZ4_stream);
             LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
             blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
@@ -713,7 +713,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_CHECKTEST(blockContinueCompressedSize != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output (%d expected vs %d actual)", expectedSize, blockContinueCompressedSize);
             FUZ_CHECKTEST(XXH32(compressedBuffer, blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
 
-            FUZ_DISPLAYTEST();
+            FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary(), but output buffer is 1 byte too short");
             LZ4_resetStream(&LZ4_stream);
             LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
             ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
-- 
cgit v0.12


From da3b5ba6f0014564b7312511e441067ba9429733 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 16 Apr 2018 23:59:42 -0700
Subject: fixed dictCtx compression

---
 lib/lz4.c | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index dca4d69..0590de4 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -602,6 +602,7 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
      * currentOffset == 0 is faster still, so we preserve that case.
      */
     if (cctx->currentOffset != 0 && tableType == byU32) {
+        DEBUGLOG(5, "LZ4_prepareTable: adding 64KB to currentOffset");
         cctx->currentOffset += 64 KB;
     }
 
@@ -636,8 +637,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         dictDirective == usingDictCtx ? dictCtx->dictionary : cctx->dictionary;
     const U32 dictSize =
         dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize;
+    const U32 dictDelta = usingDictCtx ? startIndex - dictCtx->currentOffset : 0;   /* make indexes in dictCtx comparable with index in current context */
 
-    int const maybe_ext_memSegment = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx);
+    int const maybe_extMem = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx);
     U32 const prefixIdxLimit = startIndex - dictSize;   /* used when dictDirective == dictSmall */
     const BYTE* const dictEnd = dictionary + dictSize;
     const BYTE* anchor = (const BYTE*) source;
@@ -648,7 +650,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     /* the dictCtx currentOffset is indexed on the start of the dictionary,
      * while a dictionary in the current context precedes the currentOffset */
     const BYTE* dictBase = dictDirective == usingDictCtx ?
-        dictionary + dictSize - dictCtx->currentOffset :   /* is it possible that dictCtx->currentOffset != dictCtx->dictSize ? */
+        dictionary + dictSize - dictCtx->currentOffset :   /* is it possible that dictCtx->currentOffset != dictCtx->dictSize ? Yes if the dictionary context is not reset */
         dictionary + dictSize - startIndex;
 
     BYTE* op = (BYTE*) dest;
@@ -657,6 +659,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     U32 offset = 0;
     U32 forwardH;
 
+    DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, tableType=%u", inputSize, tableType);
     /* Init conditions */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;   /* Unsupported inputSize, too large (or negative) */
     if (tableType==byPtr) assert(dictDirective==noDict);      /* only supported use case with byPtr */
@@ -731,8 +734,10 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 if (dictDirective == usingDictCtx) {
                     if (matchIndex < startIndex) {
                         /* there was no match, try the dictionary */
+                        assert(tableType == byU32);
                         matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32);
                         match = dictBase + matchIndex;
+                        matchIndex += dictDelta;   /* make dictCtx index comparable with current context */
                         lowLimit = dictionary;
                     } else {
                         match = base + matchIndex;
@@ -758,7 +763,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE);     /* too_far presumed impossible with byU16 */
 
                 if (LZ4_read32(match) == LZ4_read32(ip)) {
-                    if (maybe_ext_memSegment) offset = current - matchIndex;
+                    if (maybe_extMem) offset = current - matchIndex;
                     break;   /* match found */
                 }
 
@@ -798,7 +803,7 @@ _next_match:
          */
 
         /* Encode Offset */
-        if (maybe_ext_memSegment) {   /* static test */
+        if (maybe_extMem) {   /* static test */
             DEBUGLOG(6, "             with offset=%u  (ext if > %i)", offset, (int)(ip - (const BYTE*)source));
             assert(offset <= MAX_DISTANCE && offset > 0);
             LZ4_writeLE16(op, (U16)offset); op+=2;
@@ -878,6 +883,7 @@ _next_match:
                     matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32);
                     match = dictBase + matchIndex;
                     lowLimit = dictionary;   /* required for match length counter */
+                    matchIndex += dictDelta;
                 } else {
                     match = base + matchIndex;
                     lowLimit = (const BYTE*)source;  /* required for match length counter */
@@ -899,8 +905,7 @@ _next_match:
               && (LZ4_read32(match) == LZ4_read32(ip)) ) {
                 token=op++;
                 *token=0;
-                if (maybe_ext_memSegment)
-                    offset = current - matchIndex;
+                if (maybe_extMem) offset = current - matchIndex;
                 DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source));
                 goto _next_match;
             }
@@ -1285,7 +1290,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
     const BYTE* const dictEnd = p + dictSize;
     const BYTE* base;
 
-    DEBUGLOG(4, "LZ4_loadDict (%p into %p)", dictionary, LZ4_dict);
+    DEBUGLOG(4, "LZ4_loadDict (%i bytes from %p into %p)", dictSize, dictionary, LZ4_dict);
 
     LZ4_prepareTable(dict, 0, tableType);
 
-- 
cgit v0.12


From aedc44780468c11d90114bc1d6124af545bfc652 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Tue, 17 Apr 2018 14:01:44 -0400
Subject: Always Bump Offset by 64 KB in LZ4_loadDict()

This actually ensures the guarantee referred to in the comment in
LZ4_compress_fast_continue().
---
 lib/lz4.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 4b0efb1..0ce05da 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1173,11 +1173,18 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
 
     LZ4_prepareTable(dict, 0, tableType);
 
+    /* We always increment the offset by 64 KB, since, if the dict is longer,
+     * we truncate it to the last 64k, and if it's shorter, we still want to
+     * advance by a whole window length so we can provide the guarantee that
+     * there are only valid offsets in the window, which allows an optimization
+     * in LZ4_compress_fast_continue() where it uses noDictIssue even when the
+     * dictionary isn't a full 64k. */
+
     if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;
-    base = p - dict->currentOffset;
+    base = dictEnd - 64 KB - dict->currentOffset;
     dict->dictionary = p;
     dict->dictSize = (U32)(dictEnd - p);
-    dict->currentOffset += dict->dictSize;
+    dict->currentOffset += 64 KB;
     dict->tableType = tableType;
 
     if (dictSize < (int)HASH_UNIT) {
-- 
cgit v0.12


From 152064218361e5762fd67b5de425707fdc47095b Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 17 Apr 2018 15:29:17 -0700
Subject: fix matchIndex overflow

can happen with dictCtx
---
 lib/lz4.c | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index b426545..c799596 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -758,9 +758,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 forwardH = LZ4_hashPosition(forwardIp, tableType);
                 LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
 
-                if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue;    /* match outside of valid area */
-                if ((tableType != byU16) && (matchIndex+MAX_DISTANCE < current)) continue;  /* too far */
-                if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE);     /* too_far presumed impossible with byU16 */
+                if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue;     /* match outside of valid area */
+                if ((tableType != byU16) && (current - matchIndex > MAX_DISTANCE)) continue; /* too far - note: works even if matchIndex overflows */
+                if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE);      /* too_far presumed impossible with byU16 */
 
                 if (LZ4_read32(match) == LZ4_read32(ip)) {
                     if (maybe_extMem) offset = current - matchIndex;
@@ -861,7 +861,6 @@ _next_match:
         /* Fill table */
         LZ4_putPosition(ip-2, cctx->hashTable, tableType, base);
 
-#if 1
         /* Test next position */
         if (tableType == byPtr) {
 
@@ -901,7 +900,7 @@ _next_match:
             }
             LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
             if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1)
-              && ((tableType==byU16) ? 1 : (matchIndex+MAX_DISTANCE >= current))
+              && ((tableType==byU16) ? 1 : (current - matchIndex <= MAX_DISTANCE))
               && (LZ4_read32(match) == LZ4_read32(ip)) ) {
                 token=op++;
                 *token=0;
@@ -914,13 +913,6 @@ _next_match:
         /* Prepare next loop */
         forwardH = LZ4_hashPosition(++ip, tableType);
 
-#else
-
-        /* Prepare next loop */
-        forwardH = LZ4_hashPosition(ip, tableType);
-
-#endif
-
     }
 
 _last_literals:
-- 
cgit v0.12


From 88cca1723e76c8f5031954ba07b28447c0cb55d8 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 17 Apr 2018 16:18:37 -0700
Subject: fix dictDelta setting error

wrong test
---
 lib/lz4.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index c799596..e7553ed 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -637,7 +637,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         dictDirective == usingDictCtx ? dictCtx->dictionary : cctx->dictionary;
     const U32 dictSize =
         dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize;
-    const U32 dictDelta = usingDictCtx ? startIndex - dictCtx->currentOffset : 0;   /* make indexes in dictCtx comparable with index in current context */
+    const U32 dictDelta = (dictDirective == usingDictCtx) ? startIndex - dictCtx->currentOffset : 0;   /* make indexes in dictCtx comparable with index in current context */
 
     int const maybe_extMem = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx);
     U32 const prefixIdxLimit = startIndex - dictSize;   /* used when dictDirective == dictSmall */
-- 
cgit v0.12


From 5ad4599c5ad18d2408a6ccc545c45a36b99f0c6f Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 17 Apr 2018 16:47:56 -0700
Subject: fixed LZ4_compress_fast_extState_fastReset() in 32-bit mode

---
 lib/lz4.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index e7553ed..33aa5c7 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -983,7 +983,7 @@ int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst
                 return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
             }
         } else {
-            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+            const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
             LZ4_prepareTable(ctx, srcSize, tableType);
             return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
         }
@@ -997,7 +997,7 @@ int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst
                 return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
             }
         } else {
-            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+            const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
             LZ4_prepareTable(ctx, srcSize, tableType);
             return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
-- 
cgit v0.12


From ea6ed46fc273a7b10870caeb515622737eb1c572 Mon Sep 17 00:00:00 2001
From: Dmitrii Rodionov 
Date: Wed, 18 Apr 2018 12:20:56 +0300
Subject: Wrap likely/unlikely macroses with #ifndef

It prevent redefine error when project using lz4 has its own likely/unlikely
macroses.
---
 lib/lz4.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/lz4.c b/lib/lz4.c
index 0ce05da..1e96931 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -147,8 +147,12 @@
 #  define expect(expr,value)    (expr)
 #endif
 
+#ifndef likely
 #define likely(expr)     expect((expr) != 0, 1)
+#endif
+#ifndef unlikely
 #define unlikely(expr)   expect((expr) != 0, 0)
+#endif
 
 
 /*-************************************
-- 
cgit v0.12


From 4785bd6a3508c6e10c5cee7126fd112a0ec44599 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 18 Apr 2018 16:49:27 -0700
Subject: minor length reduction of several large lines

---
 doc/lz4frame_manual.html | 50 ++++++++++++++++++-------------
 lib/Makefile             |  4 +++
 lib/lz4.c                | 76 +++++++++++++++++++++++++++++++++---------------
 lib/lz4frame.h           | 50 ++++++++++++++++++-------------
 4 files changed, 115 insertions(+), 65 deletions(-)

diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index f0f7f5a..459bac8 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -30,9 +30,9 @@
 
 
    Error management
    
 
-
    unsigned    LZ4F_isError(LZ4F_errorCode_t code);   /**< tells if a `LZ4F_errorCode_t` function result is an error code */
+
    unsigned    LZ4F_isError(LZ4F_errorCode_t code);   /**< tells when a function result is an error code */
 

    
-
    const char* LZ4F_getErrorName(LZ4F_errorCode_t code);   /**< return error code string; useful for debugging */
+
    const char* LZ4F_getErrorName(LZ4F_errorCode_t code);   /**< return error code string; for debugging */
 

    
 
    Frame compression types
    
 
@@ -74,13 +74,13 @@
 } LZ4F_frameType_t;
 

    
 
    typedef struct {
-  LZ4F_blockSizeID_t     blockSizeID;          /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
-  LZ4F_blockMode_t       blockMode;            /* LZ4F_blockLinked, LZ4F_blockIndependent ; 0 == default */
-  LZ4F_contentChecksum_t contentChecksumFlag;  /* if enabled, frame is terminated with a 32-bits checksum of decompressed data ; 0 == disabled (default)  */
-  LZ4F_frameType_t       frameType;            /* read-only field : LZ4F_frame or LZ4F_skippableFrame */
-  unsigned long long     contentSize;          /* Size of uncompressed content ; 0 == unknown */
-  unsigned               dictID;               /* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */
-  LZ4F_blockChecksum_t   blockChecksumFlag;    /* if enabled, each block is followed by a checksum of block's compressed data ; 0 == disabled (default)  */
+  LZ4F_blockSizeID_t     blockSizeID;         /* max64KB, max256KB, max1MB, max4MB; 0 == default */
+  LZ4F_blockMode_t       blockMode;           /* LZ4F_blockLinked, LZ4F_blockIndependent; 0 == default */
+  LZ4F_contentChecksum_t contentChecksumFlag; /* 1: frame terminated with 32-bit checksum of decompressed data; 0: disabled (default) */
+  LZ4F_frameType_t       frameType;           /* read-only field : LZ4F_frame or LZ4F_skippableFrame */
+  unsigned long long     contentSize;         /* Size of uncompressed content ; 0 == unknown */
+  unsigned               dictID;              /* Dictionary ID, sent by compressor to help decoder select correct dictionary; 0 == no dictID provided */
+  LZ4F_blockChecksum_t   blockChecksumFlag;   /* 1: each block followed by a checksum of block's compressed data; 0: disabled (default) */
 } LZ4F_frameInfo_t;
 
      makes it possible to set or read frame parameters.
   It's not required to set all fields, as long as the structure was initially memset() to zero.
@@ -89,7 +89,7 @@
 
 
    typedef struct {
   LZ4F_frameInfo_t frameInfo;
-  int      compressionLevel;       /* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */
+  int      compressionLevel;       /* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */
   unsigned autoFlush;              /* 1 == always flush, to reduce usage of internal buffers */
   unsigned reserved[4];            /* must be zero for forward compatibility */
 } LZ4F_preferences_t;
@@ -165,34 +165,42 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
  
 

    
 
-
    size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr);
+
    size_t LZ4F_compressUpdate(LZ4F_cctx* cctx,
+                                       void* dstBuffer, size_t dstCapacity,
+                                 const void* srcBuffer, size_t srcSize,
+                                 const LZ4F_compressOptions_t* cOptPtr);
 
      LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
-  An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations.
+  Important rule: dstCapacity MUST be large enough to ensure operation success even in worst case situations.
   This value is provided by LZ4F_compressBound().
   If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
-  LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized.
+  LZ4F_compressUpdate() doesn't guarantee error recovery.
+  When an error occurs, compression context must be freed or resized.
  `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
  @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
            or an error code if it fails (which can be tested using LZ4F_isError())
  
 

    
 
-
    size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
+
    size_t LZ4F_flush(LZ4F_cctx* cctx,
+                              void* dstBuffer, size_t dstCapacity,
+                        const LZ4F_compressOptions_t* cOptPtr);
 
      When data must be generated and sent immediately, without waiting for a block to be completely filled,
   it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.
  `dstCapacity` must be large enough to ensure the operation will be successful.
  `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default.
- @return : number of bytes written into dstBuffer (it can be zero, which means there was no data stored within cctx)
+ @return : nb of bytes written into dstBuffer (can be zero, when there is no data stored within cctx)
            or an error code if it fails (which can be tested using LZ4F_isError())
  
 

    
 
-
    size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
+
    size_t LZ4F_compressEnd(LZ4F_cctx* cctx,
+                                    void* dstBuffer, size_t dstCapacity,
+                              const LZ4F_compressOptions_t* cOptPtr);
 
      To properly finish an LZ4 frame, invoke LZ4F_compressEnd().
   It will flush whatever data remained within `cctx` (like LZ4_flush())
   and properly finalize the frame, with an endMark and a checksum.
  `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default.
- @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled)
+ @return : nb of bytes written into dstBuffer, necessarily >= 4 (endMark),
            or an error code if it fails (which can be tested using LZ4F_isError())
   A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.
  
@@ -201,7 +209,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
 
    Decompression functions
    
 
 
    typedef struct {
-  unsigned stableDst;    /* pledge that at least 64KB+64Bytes of previously decompressed data remain unmodifed where it was decoded. This optimization skips storage operations in tmp buffers */
+  unsigned stableDst;    /* pledges that last 64KB decompressed data will remain available unmodified. This optimization skips storage operations in tmp buffers. */
   unsigned reserved[3];  /* must be set to zero for forward compatibility */
 } LZ4F_decompressOptions_t;
 

    
@@ -212,7 +220,7 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
   The function provides a pointer to an allocated and initialized LZ4F_dctx object.
   The result is an errorCode, which can be tested using LZ4F_isError().
   dctx memory can be released using LZ4F_freeDecompressionContext();
-  The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
+  Result of LZ4F_freeDecompressionContext() indicates current state of decompressionContext when being released.
   That is, it should be == 0 if decompression has been completed fully and correctly.
  
 

    
@@ -252,8 +260,8 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
   The function will read up to *srcSizePtr bytes from srcBuffer,
   and decompress data into dstBuffer, of capacity *dstSizePtr.
 
-  The number of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value).
-  The number of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value).
+  The nb of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value).
+  The nb of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value).
 
   The function does not necessarily read all input bytes, so always check value in *srcSizePtr.
   Unconsumed source data must be presented again in subsequent invocations.
diff --git a/lib/Makefile b/lib/Makefile
index dd33f50..7b31239 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -116,6 +116,10 @@ clean:
 #-----------------------------------------------------------------------------
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
 
+.PHONY: listL120
+listL120:  # extract lines >= 120 characters in *.{c,h}, by Takayuki Matsuoka (note : $$, for Makefile compatibility)
+	find . -type f -name '*.c' -o -name '*.h' | while read -r filename; do awk 'length > 120 {print FILENAME "(" FNR "): " $$0}' $$filename; done
+
 DESTDIR     ?=
 # directory variables : GNU conventions prefer lowercase
 # see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
diff --git a/lib/lz4.c b/lib/lz4.c
index 33aa5c7..af3221c 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -69,7 +69,9 @@
  * Prefer these methods in priority order (0 > 1 > 2)
  */
 #ifndef LZ4_FORCE_MEMORY_ACCESS   /* can be defined externally */
-#  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
+#  if defined(__GNUC__) && \
+  ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \
+  || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
 #    define LZ4_FORCE_MEMORY_ACCESS 2
 #  elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || defined(__GNUC__)
 #    define LZ4_FORCE_MEMORY_ACCESS 1
@@ -80,7 +82,7 @@
  * LZ4_FORCE_SW_BITCOUNT
  * Define this parameter if your target system or compiler does not support hardware bit count
  */
-#if defined(_MSC_VER) && defined(_WIN32_WCE)   /* Visual Studio for Windows CE does not support Hardware bit count */
+#if defined(_MSC_VER) && defined(_WIN32_WCE)   /* Visual Studio for WinCE doesn't support Hardware bit count */
 #  define LZ4_FORCE_SW_BITCOUNT
 #endif
 
@@ -318,7 +320,7 @@ static const int LZ4_minLength = (MFLIMIT+1);
 #  endif
 #endif
 
-#define LZ4_STATIC_ASSERT(c)   { enum { LZ4_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
+#define LZ4_STATIC_ASSERT(c)   { enum { LZ4_static_assert = 1/(int)(!!(c)) }; }   /* use after variable declarations */
 
 #if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2)
 #  include 
@@ -529,7 +531,9 @@ static void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t cons
     }
 }
 
-static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase)
+static void LZ4_putPositionOnHash(const BYTE* p, U32 h,
+                                  void* tableBase, tableType_t const tableType,
+                            const BYTE* srcBase)
 {
     switch (tableType)
     {
@@ -555,8 +559,16 @@ LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_
 static U32 LZ4_getIndexOnHash(U32 h, const void* tableBase, tableType_t tableType)
 {
     LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2);
-    if (tableType == byU32) { const U32* const hashTable = (const U32*) tableBase; assert(h < (1U << (LZ4_MEMORY_USAGE-2))); return hashTable[h]; }
-    if (tableType == byU16) { const U16* const hashTable = (const U16*) tableBase; assert(h < (1U << (LZ4_MEMORY_USAGE-1))); return hashTable[h]; }
+    if (tableType == byU32) {
+        const U32* const hashTable = (const U32*) tableBase;
+        assert(h < (1U << (LZ4_MEMORY_USAGE-2)));
+        return hashTable[h];
+    }
+    if (tableType == byU16) {
+        const U16* const hashTable = (const U16*) tableBase;
+        assert(h < (1U << (LZ4_MEMORY_USAGE-1)));
+        return hashTable[h];
+    }
     assert(0); return 0;  /* forbidden case */
 }
 
@@ -567,7 +579,9 @@ static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType
     { const U16* const hashTable = (const U16*) tableBase; return hashTable[h] + srcBase; }   /* default, to ensure a return */
 }
 
-LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, const void* tableBase, tableType_t tableType, const BYTE* srcBase)
+LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p,
+                                             const void* tableBase, tableType_t tableType,
+                                             const BYTE* srcBase)
 {
     U32 const h = LZ4_hashPosition(p, tableType);
     return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
@@ -650,7 +664,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     /* the dictCtx currentOffset is indexed on the start of the dictionary,
      * while a dictionary in the current context precedes the currentOffset */
     const BYTE* dictBase = dictDirective == usingDictCtx ?
-        dictionary + dictSize - dictCtx->currentOffset :   /* is it possible that dictCtx->currentOffset != dictCtx->dictSize ? Yes if the dictionary context is not reset */
+        dictionary + dictSize - dictCtx->currentOffset :
         dictionary + dictSize - startIndex;
 
     BYTE* op = (BYTE*) dest;
@@ -667,7 +681,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0);
 
-    if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;   /* Size too large (not within 64K limit) */
+    if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;  /* Size too large (not within 64K limit) */
 
     /* Update context state */
     if (dictDirective == usingDictCtx) {
@@ -681,7 +695,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     cctx->currentOffset += (U32)inputSize;
     cctx->tableType = tableType;
 
-    if (inputSizehashTable, tableType, base);
@@ -790,7 +804,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
             /* Copy Literals */
             LZ4_wildCopy(op, anchor, op+litLength);
             op+=litLength;
-            DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", (int)(anchor-(const BYTE*)source), litLength, (int)(ip-(const BYTE*)source));
+            DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i",
+                        (int)(anchor-(const BYTE*)source), litLength, (int)(ip-(const BYTE*)source));
         }
 
 _next_match:
@@ -905,7 +920,8 @@ _next_match:
                 token=op++;
                 *token=0;
                 if (maybe_extMem) offset = current - matchIndex;
-                DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source));
+                DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i",
+                            (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source));
                 goto _next_match;
             }
         }
@@ -1649,19 +1665,25 @@ _output_error:
 LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize)
 {
-    return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, (BYTE*)dest, NULL, 0);
+    return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize,
+                                  endOnInputSize, full, 0, noDict,
+                                  (BYTE*)dest, NULL, 0);
 }
 
 LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize)
 {
-    return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0);
+    return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize,
+                                  endOnInputSize, partial, targetOutputSize,
+                                  noDict, (BYTE*)dest, NULL, 0);
 }
 
 LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
 {
-    return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB);
+    return LZ4_decompress_generic(source, dest, 0, originalSize,
+                                  endOnOutputSize, full, 0, withPrefix64k,
+                                  (BYTE*)(dest - 64 KB), NULL, 64 KB);
 }
 
 
@@ -1803,12 +1825,18 @@ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compres
 *  Obsolete Functions
 ***************************************************/
 /* obsolete compression functions */
-int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); }
-int LZ4_compress(const char* source, char* dest, int inputSize) { return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); }
-int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); }
-int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); }
-int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); }
-int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) { return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); }
+int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) {
+    return LZ4_compress_default(source, dest, inputSize, maxOutputSize); }
+int LZ4_compress(const char* source, char* dest, int inputSize) {
+    return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); }
+int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) {
+    return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); }
+int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) {
+    return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); }
+int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) {
+    return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); }
+int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) {
+    return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); }
 
 /*
 These function names are deprecated and should no longer be used.
@@ -1816,8 +1844,10 @@ They are only provided here for compatibility with older user programs.
 - LZ4_uncompress is totally equivalent to LZ4_decompress_fast
 - LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe
 */
-int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); }
-int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
+int LZ4_uncompress (const char* source, char* dest, int outputSize) {
+    return LZ4_decompress_fast(source, dest, outputSize); }
+int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) {
+    return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
 
 /* Obsolete Streaming functions */
 
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 9efaca0..e80b1a1 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -93,8 +93,8 @@ extern "C" {
  **************************************/
 typedef size_t LZ4F_errorCode_t;
 
-LZ4FLIB_API unsigned    LZ4F_isError(LZ4F_errorCode_t code);   /**< tells if a `LZ4F_errorCode_t` function result is an error code */
-LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code);   /**< return error code string; useful for debugging */
+LZ4FLIB_API unsigned    LZ4F_isError(LZ4F_errorCode_t code);   /**< tells when a function result is an error code */
+LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code);   /**< return error code string; for debugging */
 
 
 /*-************************************
@@ -162,13 +162,13 @@ typedef LZ4F_contentChecksum_t contentChecksum_t;
  *  It's not required to set all fields, as long as the structure was initially memset() to zero.
  *  For all fields, 0 sets it to default value */
 typedef struct {
-  LZ4F_blockSizeID_t     blockSizeID;          /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
-  LZ4F_blockMode_t       blockMode;            /* LZ4F_blockLinked, LZ4F_blockIndependent ; 0 == default */
-  LZ4F_contentChecksum_t contentChecksumFlag;  /* if enabled, frame is terminated with a 32-bits checksum of decompressed data ; 0 == disabled (default)  */
-  LZ4F_frameType_t       frameType;            /* read-only field : LZ4F_frame or LZ4F_skippableFrame */
-  unsigned long long     contentSize;          /* Size of uncompressed content ; 0 == unknown */
-  unsigned               dictID;               /* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */
-  LZ4F_blockChecksum_t   blockChecksumFlag;    /* if enabled, each block is followed by a checksum of block's compressed data ; 0 == disabled (default)  */
+  LZ4F_blockSizeID_t     blockSizeID;         /* max64KB, max256KB, max1MB, max4MB; 0 == default */
+  LZ4F_blockMode_t       blockMode;           /* LZ4F_blockLinked, LZ4F_blockIndependent; 0 == default */
+  LZ4F_contentChecksum_t contentChecksumFlag; /* 1: frame terminated with 32-bit checksum of decompressed data; 0: disabled (default) */
+  LZ4F_frameType_t       frameType;           /* read-only field : LZ4F_frame or LZ4F_skippableFrame */
+  unsigned long long     contentSize;         /* Size of uncompressed content ; 0 == unknown */
+  unsigned               dictID;              /* Dictionary ID, sent by compressor to help decoder select correct dictionary; 0 == no dictID provided */
+  LZ4F_blockChecksum_t   blockChecksumFlag;   /* 1: each block followed by a checksum of block's compressed data; 0: disabled (default) */
 } LZ4F_frameInfo_t;
 
 /*! LZ4F_preferences_t :
@@ -177,7 +177,7 @@ typedef struct {
  *  All reserved fields must be set to zero. */
 typedef struct {
   LZ4F_frameInfo_t frameInfo;
-  int      compressionLevel;       /* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */
+  int      compressionLevel;       /* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */
   unsigned autoFlush;              /* 1 == always flush, to reduce usage of internal buffers */
   unsigned reserved[4];            /* must be zero for forward compatibility */
 } LZ4F_preferences_t;
@@ -264,36 +264,44 @@ LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t*
 
 /*! LZ4F_compressUpdate() :
  *  LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
- *  An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations.
+ *  Important rule: dstCapacity MUST be large enough to ensure operation success even in worst case situations.
  *  This value is provided by LZ4F_compressBound().
  *  If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
- *  LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized.
+ *  LZ4F_compressUpdate() doesn't guarantee error recovery.
+ *  When an error occurs, compression context must be freed or resized.
  * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
  * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
  *           or an error code if it fails (which can be tested using LZ4F_isError())
  */
-LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr);
+LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx,
+                                       void* dstBuffer, size_t dstCapacity,
+                                 const void* srcBuffer, size_t srcSize,
+                                 const LZ4F_compressOptions_t* cOptPtr);
 
 /*! LZ4F_flush() :
  *  When data must be generated and sent immediately, without waiting for a block to be completely filled,
  *  it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.
  * `dstCapacity` must be large enough to ensure the operation will be successful.
  * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default.
- * @return : number of bytes written into dstBuffer (it can be zero, which means there was no data stored within cctx)
+ * @return : nb of bytes written into dstBuffer (can be zero, when there is no data stored within cctx)
  *           or an error code if it fails (which can be tested using LZ4F_isError())
  */
-LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
+LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx,
+                              void* dstBuffer, size_t dstCapacity,
+                        const LZ4F_compressOptions_t* cOptPtr);
 
 /*! LZ4F_compressEnd() :
  *  To properly finish an LZ4 frame, invoke LZ4F_compressEnd().
  *  It will flush whatever data remained within `cctx` (like LZ4_flush())
  *  and properly finalize the frame, with an endMark and a checksum.
  * `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default.
- * @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled)
+ * @return : nb of bytes written into dstBuffer, necessarily >= 4 (endMark),
  *           or an error code if it fails (which can be tested using LZ4F_isError())
  *  A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.
  */
-LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
+LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx,
+                                    void* dstBuffer, size_t dstCapacity,
+                              const LZ4F_compressOptions_t* cOptPtr);
 
 
 /*-*********************************
@@ -303,7 +311,7 @@ typedef struct LZ4F_dctx_s LZ4F_dctx;   /* incomplete type */
 typedef LZ4F_dctx* LZ4F_decompressionContext_t;   /* compatibility with previous API versions */
 
 typedef struct {
-  unsigned stableDst;    /* pledge that at least 64KB+64Bytes of previously decompressed data remain unmodifed where it was decoded. This optimization skips storage operations in tmp buffers */
+  unsigned stableDst;    /* pledges that last 64KB decompressed data will remain available unmodified. This optimization skips storage operations in tmp buffers. */
   unsigned reserved[3];  /* must be set to zero for forward compatibility */
 } LZ4F_decompressOptions_t;
 
@@ -316,7 +324,7 @@ typedef struct {
  *  The function provides a pointer to an allocated and initialized LZ4F_dctx object.
  *  The result is an errorCode, which can be tested using LZ4F_isError().
  *  dctx memory can be released using LZ4F_freeDecompressionContext();
- *  The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
+ *  Result of LZ4F_freeDecompressionContext() indicates current state of decompressionContext when being released.
  *  That is, it should be == 0 if decompression has been completed fully and correctly.
  */
 LZ4FLIB_API LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version);
@@ -357,8 +365,8 @@ LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
  *  The function will read up to *srcSizePtr bytes from srcBuffer,
  *  and decompress data into dstBuffer, of capacity *dstSizePtr.
  *
- *  The number of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value).
- *  The number of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value).
+ *  The nb of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value).
+ *  The nb of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value).
  *
  *  The function does not necessarily read all input bytes, so always check value in *srcSizePtr.
  *  Unconsumed source data must be presented again in subsequent invocations.
-- 
cgit v0.12


From 46058d71aa97cf29690f7638179104985c28d354 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 19 Apr 2018 10:50:40 -0700
Subject: modified indentation for consistency

---
 lib/lz4.c | 50 +++++++++++++++++++++++++++++++++-----------------
 1 file changed, 33 insertions(+), 17 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index af3221c..10674a3 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1825,29 +1825,45 @@ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compres
 *  Obsolete Functions
 ***************************************************/
 /* obsolete compression functions */
-int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) {
-    return LZ4_compress_default(source, dest, inputSize, maxOutputSize); }
-int LZ4_compress(const char* source, char* dest, int inputSize) {
-    return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); }
-int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) {
-    return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); }
-int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) {
-    return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); }
-int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) {
-    return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); }
-int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) {
-    return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); }
+int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize)
+{
+    return LZ4_compress_default(source, dest, inputSize, maxOutputSize);
+}
+int LZ4_compress(const char* source, char* dest, int inputSize)
+{
+    return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize));
+}
+int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize)
+{
+    return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1);
+}
+int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize)
+{
+    return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1);
+}
+int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int dstCapacity)
+{
+    return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, dstCapacity, 1);
+}
+int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize)
+{
+    return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1);
+}
 
 /*
-These function names are deprecated and should no longer be used.
+These decompression functions are deprecated and should no longer be used.
 They are only provided here for compatibility with older user programs.
 - LZ4_uncompress is totally equivalent to LZ4_decompress_fast
 - LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe
 */
-int LZ4_uncompress (const char* source, char* dest, int outputSize) {
-    return LZ4_decompress_fast(source, dest, outputSize); }
-int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) {
-    return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
+int LZ4_uncompress (const char* source, char* dest, int outputSize)
+{
+    return LZ4_decompress_fast(source, dest, outputSize);
+}
+int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize)
+{
+    return LZ4_decompress_safe(source, dest, isize, maxOutputSize);
+}
 
 /* Obsolete Streaming functions */
 
-- 
cgit v0.12