From c14a342cc2ed5c963213571ab412657bdcf2b899 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 6 Oct 2014 00:48:45 +0100 Subject: Fixed : LZ4F_compressBound() using null preferencesPtr Updated : frametest, to check LZ4F_compressBound() using null preferencesPtr --- lz4frame.c | 21 +++++++++++---------- programs/frametest.c | 17 ++++++++--------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lz4frame.c b/lz4frame.c index 3a750c0..93069e0 100644 --- a/lz4frame.c +++ b/lz4frame.c @@ -122,7 +122,7 @@ typedef struct BYTE* tmpBuff; BYTE* tmpIn; size_t tmpInSize; - XXH32_stateSpace_t xxh; + XXH32_state_t xxh; LZ4_stream_t lz4ctx; } LZ4F_cctx_internal_t; @@ -143,7 +143,7 @@ typedef struct BYTE* tmpOut; size_t tmpOutSize; size_t tmpOutStart; - XXH32_stateSpace_t xxh; + XXH32_state_t xxh; BYTE header[8]; } LZ4F_dctx_internal_t; @@ -383,7 +383,7 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds } cctxPtr->tmpIn = cctxPtr->tmpBuff; cctxPtr->tmpInSize = 0; - XXH32_resetState(&(cctxPtr->xxh), 0); + XXH32_reset(&(cctxPtr->xxh), 0); LZ4_resetStream(&(cctxPtr->lz4ctx)); /* Magic Number */ @@ -413,13 +413,14 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds * */ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { - LZ4F_frameInfo_t* frameInfoPtr = (LZ4F_frameInfo_t*)preferencesPtr; /* works because prefs starts with frameInfo */ - blockSizeID_t bid = (frameInfoPtr==NULL) ? LZ4F_BLOCKSIZEID_DEFAULT : frameInfoPtr->blockSizeID; + const LZ4F_preferences_t prefsNull = { 0 }; + const LZ4F_preferences_t* prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; + blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID; size_t blockSize = LZ4F_getBlockSize(bid); unsigned nbBlocks = (unsigned)(srcSize / blockSize) + 1; - size_t lastBlockSize = preferencesPtr->autoFlush ? srcSize % blockSize : blockSize; + size_t lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize; size_t blockInfo = 4; /* default, without block CRC option */ - size_t frameEnd = 4 + (frameInfoPtr->contentChecksumFlag*4); + size_t frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); size_t result = (blockInfo * nbBlocks) + (blockSize * (nbBlocks-1)) + lastBlockSize + frameEnd; return result; @@ -628,7 +629,7 @@ size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstB if (cctxPtr->prefs.frameInfo.contentChecksumFlag == contentChecksumEnabled) { - U32 xxh = XXH32_intermediateDigest(&(cctxPtr->xxh)); + U32 xxh = XXH32_digest(&(cctxPtr->xxh)); LZ4F_writeLE32(dstPtr, xxh); dstPtr+=4; /* content Checksum */ } @@ -721,7 +722,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPt dctxPtr->maxBlockSize = LZ4F_getBlockSize(blockSizeID); /* init */ - if (contentChecksumFlag) XXH32_resetState(&(dctxPtr->xxh), 0); + if (contentChecksumFlag) XXH32_reset(&(dctxPtr->xxh), 0); /* alloc */ bufferNeeded = dctxPtr->maxBlockSize + ((dctxPtr->frameInfo.blockMode==blockLinked) * 128 KB); @@ -1220,7 +1221,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, case dstage_checkSuffix: { U32 readCRC = LZ4F_readLE32(selectedIn); - U32 resultCRC = XXH32_intermediateDigest(&(dctxPtr->xxh)); + U32 resultCRC = XXH32_digest(&(dctxPtr->xxh)); if (readCRC != resultCRC) return (size_t)-ERROR_checksum_invalid; nextSrcSizeHint = 0; dctxPtr->dStage = dstage_getHeader; diff --git a/programs/frametest.c b/programs/frametest.c index 3d19ef4..343a8ab 100644 --- a/programs/frametest.c +++ b/programs/frametest.c @@ -131,7 +131,6 @@ static U32 FUZ_GetMilliSpan(U32 nTimeStart) } - # define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) unsigned int FUZ_rand(unsigned int* src) { @@ -388,7 +387,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi LZ4F_decompressionContext_t dCtx = NULL; LZ4F_compressionContext_t cCtx = NULL; size_t result; - XXH64_stateSpace_t xxh64; + XXH64_state_t xxh64; # define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } @@ -424,21 +423,21 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi size_t srcStart = FUZ_rand(&randState) % (srcDataLength - srcSize); size_t cSize; U64 crcOrig, crcDecoded; + LZ4F_preferences_t* prefsPtr = &prefs; (void)FUZ_rand(&coreRand); // update rand seed prefs.frameInfo.blockMode = BMId; prefs.frameInfo.blockSizeID = BSId; prefs.frameInfo.contentChecksumFlag = CCflag; prefs.autoFlush = autoflush; + if ((FUZ_rand(&randState)&0xF) == 1) prefsPtr = NULL; DISPLAYUPDATE(2, "\r%5u ", testNb); crcOrig = XXH64((BYTE*)srcBuffer+srcStart, (U32)srcSize, 1); if ((FUZ_rand(&randState)&0xF) == 2) { - LZ4F_preferences_t* framePrefs = &prefs; - if ((FUZ_rand(&randState)&7) == 1) framePrefs = NULL; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(srcSize, framePrefs), (char*)srcBuffer + srcStart, srcSize, framePrefs); + cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(srcSize, prefsPtr), (char*)srcBuffer + srcStart, srcSize, prefsPtr); CHECK(LZ4F_isError(cSize), "LZ4F_compressFrame failed : error %i (%s)", (int)cSize, LZ4F_getErrorName(cSize)); } else @@ -448,14 +447,14 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi BYTE* op = compressedBuffer; BYTE* const oend = op + LZ4F_compressFrameBound(srcDataLength, NULL); unsigned maxBits = FUZ_highbit((U32)srcSize); - result = LZ4F_compressBegin(cCtx, op, oend-op, &prefs); + result = LZ4F_compressBegin(cCtx, op, oend-op, prefsPtr); CHECK(LZ4F_isError(result), "Compression header failed (error %i)", (int)result); op += result; while (ip < iend) { unsigned nbBitsSeg = FUZ_rand(&randState) % maxBits; size_t iSize = (FUZ_rand(&randState) & ((1< (size_t)(iend-ip)) iSize = iend-ip; cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1); @@ -486,7 +485,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi unsigned maxBits = FUZ_highbit((U32)cSize); unsigned nonContiguousDst = (FUZ_rand(&randState) & 3) == 1; nonContiguousDst += FUZ_rand(&randState) & nonContiguousDst; /* 0=>0; 1=>1,2 */ - XXH64_resetState(&xxh64, 1); + XXH64_reset(&xxh64, 1); while (ip < iend) { unsigned nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1; @@ -510,7 +509,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi if (nonContiguousDst==2) op = decodedBuffer; // overwritten destination } CHECK(result != 0, "Frame decompression failed (error %i)", (int)result); - crcDecoded = XXH64_intermediateDigest(&xxh64); + crcDecoded = XXH64_digest(&xxh64); if (crcDecoded != crcOrig) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst); CHECK(crcDecoded != crcOrig, "Decompression corruption"); } -- cgit v0.12 From 72b51859e03c2a09b4e6f3a3d5d4783a8a99f6d4 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 6 Oct 2014 01:10:29 +0100 Subject: update distribution builder "make dist" update test, to also verify "make dist" --- Makefile | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index b21ae2f..a46cb41 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ # ################################################################ # Version numbers -VERSION=123 +VERSION=124 export RELEASE=r$(VERSION) LIBVER_MAJOR=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz4.h` LIBVER_MINOR=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz4.h` @@ -71,7 +71,8 @@ else SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER) endif -TEXT = lz4.c lz4.h lz4hc.c lz4hc.h \ +TEXT = lz4.c lz4.h lz4hc.c lz4hc.h \ + lz4frame.c lz4frame.h xxhash.c xxhash.h \ liblz4.pc.in Makefile \ lz4_format_description.txt NEWS LICENSE README.md \ cmake_unofficial/CMakeLists.txt \ @@ -79,10 +80,12 @@ TEXT = lz4.c lz4.h lz4hc.c lz4hc.h \ $(PRGDIR)/datagen.c $(PRGDIR)/fuzzer.c \ $(PRGDIR)/lz4io.c $(PRGDIR)/lz4io.h \ $(PRGDIR)/bench.c $(PRGDIR)/bench.h \ - $(PRGDIR)/xxhash.c $(PRGDIR)/xxhash.h \ $(PRGDIR)/lz4.1 $(PRGDIR)/lz4c.1 $(PRGDIR)/lz4cat.1 \ - $(PRGDIR)/Makefile $(PRGDIR)/COPYING -NONTEXT = LZ4_Streaming_Format.odt + $(PRGDIR)/Makefile $(PRGDIR)/COPYING \ + LZ4_Framing_Format.html +NONTEXT = images/image00.png images/image01.png images/image02.png \ + images/image03.png images/image04.png images/image05.png \ + images/image06.png SOURCES = $(TEXT) $(NONTEXT) @@ -149,6 +152,7 @@ uninstall: dist: clean @install -dD -m 700 lz4-$(RELEASE)/programs/ @install -dD -m 700 lz4-$(RELEASE)/cmake_unofficial/ + @install -dD -m 700 lz4-$(RELEASE)/images/ @for f in $(TEXT); do \ tr -d '\r' < $$f > .tmp; \ install -m 600 .tmp lz4-$(RELEASE)/$$f; \ @@ -163,6 +167,7 @@ dist: clean @echo Distribution $(DISTRIBNAME) built test: + make dist @cd examples; $(MAKE) -e $@ @cd $(PRGDIR); $(MAKE) -e $@ -- cgit v0.12 From 2ab6f9a387de9d507e38ae63f6da0f9b9b9a58d8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 6 Oct 2014 11:13:56 +0100 Subject: Updated : xxHash to r37 --- NEWS | 4 + programs/lz4io.c | 28 ++-- xxhash.c | 470 +++++++++++++++++++++++++++++++++++-------------------- xxhash.h | 98 +++++------- 4 files changed, 360 insertions(+), 240 deletions(-) diff --git a/NEWS b/NEWS index 41d5762..7edee54 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +r124: +Fix : LZ4F_compressBound() using NULL preferencesPtr +Updated : xxHash, to r37 + r123: Added : experimental lz4frame API, thanks to Takayuki Matsuoka and Christopher Jackson for testings Fix : s390x support, thanks to Nobuhiro Iwamatsu diff --git a/programs/lz4io.c b/programs/lz4io.c index 20520d6..6de5206 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -409,7 +409,7 @@ static int compress_file_blockDependency(char* input_filename, char* output_file clock_t start, end; unsigned int blockSize, inputBufferSize; size_t sizeCheck, header_size; - void* streamChecksumState=NULL; + XXH32_state_t streamCRC; // Init start = clock(); @@ -440,7 +440,7 @@ static int compress_file_blockDependency(char* input_filename, char* output_file if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory"); in_blockStart = in_buff + 64 KB; if (compressionlevel>=3) in_blockStart = in_buff; - if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED); + if (streamChecksum) XXH32_reset(&streamCRC, LZ4S_CHECKSUM_SEED); ctx = initFunction(in_buff); // Write Archive Header @@ -469,7 +469,7 @@ static int compress_file_blockDependency(char* input_filename, char* output_file if( inSize==0 ) break; // No more input : end of compression filesize += inSize; DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20)); - if (streamChecksum) XXH32_update(streamChecksumState, in_blockStart, inSize); + if (streamChecksum) XXH32_update(&streamCRC, in_blockStart, inSize); // Compress Block outSize = compressionFunction(ctx, in_blockStart, out_buff+4, inSize, inSize-1, compressionlevel); @@ -521,7 +521,7 @@ static int compress_file_blockDependency(char* input_filename, char* output_file compressedfilesize += 4; if (streamChecksum) { - unsigned int checksum = XXH32_digest(streamChecksumState); + unsigned int checksum = XXH32_digest(&streamCRC); * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); sizeCheck = fwrite(out_buff, 1, 4, foutput); if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum"); @@ -566,7 +566,7 @@ int LZ4IO_compressFilename(char* input_filename, char* output_filename, int comp clock_t start, end; int blockSize; size_t sizeCheck, header_size, readSize; - void* streamChecksumState=NULL; + XXH32_state_t streamCRC; // Branch out if (blockIndependence==0) return compress_file_blockDependency(input_filename, output_filename, compressionLevel); @@ -584,7 +584,7 @@ int LZ4IO_compressFilename(char* input_filename, char* output_filename, int comp out_buff = (char*)malloc(blockSize+CACHELINE); headerBuffer = (char*)malloc(LZ4S_MAXHEADERSIZE); if (!in_buff || !out_buff || !(headerBuffer)) EXM_THROW(31, "Allocation error : not enough memory"); - if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED); + if (streamChecksum) XXH32_reset(&streamCRC, LZ4S_CHECKSUM_SEED); // Write Archive Header *(unsigned int*)headerBuffer = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention @@ -613,7 +613,7 @@ int LZ4IO_compressFilename(char* input_filename, char* output_filename, int comp filesize += readSize; DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20)); - if (streamChecksum) XXH32_update(streamChecksumState, in_buff, (int)readSize); + if (streamChecksum) XXH32_update(&streamCRC, in_buff, (int)readSize); // Compress Block outSize = compressionFunction(in_buff, out_buff+4, (int)readSize, (int)readSize-1, compressionLevel); @@ -662,8 +662,8 @@ int LZ4IO_compressFilename(char* input_filename, char* output_filename, int comp compressedfilesize += 4; if (streamChecksum) { - unsigned int checksum = XXH32_digest(streamChecksumState); - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); + unsigned int checksum = XXH32_digest(&streamCRC); + *(unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); sizeCheck = fwrite(out_buff, 1, 4, foutput); if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum"); compressedfilesize += 4; @@ -755,7 +755,7 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) unsigned int maxBlockSize; size_t sizeCheck; int blockChecksumFlag, streamChecksumFlag, blockIndependenceFlag; - void* streamChecksumState=NULL; + XXH32_state_t streamCRC; int (*decompressionFunction)(LZ4_streamDecode_t* ctx, const char* src, char* dst, int cSize, int maxOSize) = LZ4_decompress_safe_continue; LZ4_streamDecode_t ctx; @@ -805,7 +805,7 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) out_start = out_buff; out_end = out_start + outBuffSize; if (!in_buff || !out_buff) EXM_THROW(70, "Allocation error : not enough memory"); - if (streamChecksumFlag) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED); + if (streamChecksumFlag) XXH32_reset(&streamCRC, LZ4S_CHECKSUM_SEED); } // Main Loop @@ -843,7 +843,7 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) sizeCheck = fwrite(in_buff, 1, blockSize, foutput); if (sizeCheck != (size_t)blockSize) EXM_THROW(76, "Write error : cannot write data block"); filesize += blockSize; - if (streamChecksumFlag) XXH32_update(streamChecksumState, in_buff, blockSize); + if (streamChecksumFlag) XXH32_update(&streamCRC, in_buff, blockSize); if (!blockIndependenceFlag) { // handle dictionary for streaming @@ -859,7 +859,7 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) decodedBytes = decompressionFunction(&ctx, in_buff, out_start, blockSize, maxBlockSize); if (decodedBytes < 0) EXM_THROW(77, "Decoding Failed ! Corrupted input detected !"); filesize += decodedBytes; - if (streamChecksumFlag) XXH32_update(streamChecksumState, out_start, decodedBytes); + if (streamChecksumFlag) XXH32_update(&streamCRC, out_start, decodedBytes); // Write Block sizeCheck = fwrite(out_start, 1, decodedBytes, foutput); @@ -872,7 +872,7 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) // Stream Checksum if (streamChecksumFlag) { - unsigned int checksum = XXH32_digest(streamChecksumState); + unsigned int checksum = XXH32_digest(&streamCRC); unsigned int readChecksum; sizeCheck = fread(&readChecksum, 1, 4, finput); if (sizeCheck != 4) EXM_THROW(74, "Read error : cannot read stream checksum"); diff --git a/xxhash.c b/xxhash.c index be48370..529d69c 100644 --- a/xxhash.c +++ b/xxhash.c @@ -28,6 +28,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - xxHash source repository : http://code.google.com/p/xxhash/ +- public discussion board : https://groups.google.com/forum/#!forum/lz4c */ @@ -80,14 +81,23 @@ You can contact the author at : // Includes & Memory related functions //************************************** #include "xxhash.h" -// Modify the local functions below should you wish to use some other memory related routines +// Modify the local functions below should you wish to use some other memory routines // for malloc(), free() #include -FORCE_INLINE void* XXH_malloc(size_t s) { return malloc(s); } -FORCE_INLINE void XXH_free (void* p) { free(p); } +FORCE_INLINE void* XXH_malloc(size_t s) +{ + return malloc(s); +} +FORCE_INLINE void XXH_free (void* p) +{ + free(p); +} // for memcpy() #include -FORCE_INLINE void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } +FORCE_INLINE void* XXH_memcpy(void* dest, const void* src, size_t size) +{ + return memcpy(dest,src,size); +} //************************************** @@ -95,17 +105,17 @@ FORCE_INLINE void* XXH_memcpy(void* dest, const void* src, size_t size) { return //************************************** #if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 # include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; +typedef uint8_t BYTE; +typedef uint16_t U16; +typedef uint32_t U32; +typedef int32_t S32; +typedef uint64_t U64; #else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; +typedef unsigned char BYTE; +typedef unsigned short U16; +typedef unsigned int U32; +typedef signed int S32; +typedef unsigned long long U64; #endif #if defined(__GNUC__) && !defined(XXH_USE_UNALIGNED_ACCESS) @@ -122,8 +132,14 @@ FORCE_INLINE void* XXH_memcpy(void* dest, const void* src, size_t size) { return # endif #endif -typedef struct _U32_S { U32 v; } _PACKED U32_S; -typedef struct _U64_S { U64 v; } _PACKED U64_S; +typedef struct _U32_S +{ + U32 v; +} _PACKED U32_S; +typedef struct _U64_S +{ + U64 v; +} _PACKED U64_S; #if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) # pragma pack(pop) @@ -154,12 +170,15 @@ typedef struct _U64_S { U64 v; } _PACKED U64_S; # define XXH_swap32 __builtin_bswap32 # define XXH_swap64 __builtin_bswap64 #else -static inline U32 XXH_swap32 (U32 x) { +static inline U32 XXH_swap32 (U32 x) +{ return ((x << 24) & 0xff000000 ) | - ((x << 8) & 0x00ff0000 ) | - ((x >> 8) & 0x0000ff00 ) | - ((x >> 24) & 0x000000ff );} -static inline U64 XXH_swap64 (U64 x) { + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +static inline U64 XXH_swap64 (U64 x) +{ return ((x << 56) & 0xff00000000000000ULL) | ((x << 40) & 0x00ff000000000000ULL) | ((x << 24) & 0x0000ff0000000000ULL) | @@ -167,7 +186,8 @@ static inline U64 XXH_swap64 (U64 x) { ((x >> 8) & 0x00000000ff000000ULL) | ((x >> 24) & 0x0000000000ff0000ULL) | ((x >> 40) & 0x000000000000ff00ULL) | - ((x >> 56) & 0x00000000000000ffULL);} + ((x >> 56) & 0x00000000000000ffULL); +} #endif @@ -191,7 +211,7 @@ static inline U64 XXH_swap64 (U64 x) { //************************************** typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; #ifndef XXH_CPU_LITTLE_ENDIAN // It is possible to define XXH_CPU_LITTLE_ENDIAN externally, for example using a compiler switch - static const int one = 1; +static const int one = 1; # define XXH_CPU_LITTLE_ENDIAN (*(char*)(&one)) #endif @@ -215,7 +235,10 @@ FORCE_INLINE U32 XXH_readLE32_align(const U32* ptr, XXH_endianess endian, XXH_al return endian==XXH_littleEndian ? *ptr : XXH_swap32(*ptr); } -FORCE_INLINE U32 XXH_readLE32(const U32* ptr, XXH_endianess endian) { return XXH_readLE32_align(ptr, endian, XXH_unaligned); } +FORCE_INLINE U32 XXH_readLE32(const U32* ptr, XXH_endianess endian) +{ + return XXH_readLE32_align(ptr, endian, XXH_unaligned); +} FORCE_INLINE U64 XXH_readLE64_align(const U64* ptr, XXH_endianess endian, XXH_alignment align) { @@ -225,13 +248,16 @@ FORCE_INLINE U64 XXH_readLE64_align(const U64* ptr, XXH_endianess endian, XXH_al return endian==XXH_littleEndian ? *ptr : XXH_swap64(*ptr); } -FORCE_INLINE U64 XXH_readLE64(const U64* ptr, XXH_endianess endian) { return XXH_readLE64_align(ptr, endian, XXH_unaligned); } +FORCE_INLINE U64 XXH_readLE64(const U64* ptr, XXH_endianess endian) +{ + return XXH_readLE64_align(ptr, endian, XXH_unaligned); +} //**************************** // Simple Hash Functions //**************************** -FORCE_INLINE U32 XXH32_endian_align(const void* input, unsigned int len, U32 seed, XXH_endianess endian, XXH_alignment align) +FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) { const BYTE* p = (const BYTE*)input; const BYTE* bEnd = p + len; @@ -239,7 +265,11 @@ FORCE_INLINE U32 XXH32_endian_align(const void* input, unsigned int len, U32 see #define XXH_get32bits(p) XXH_readLE32_align((const U32*)p, endian, align) #ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) { len=0; bEnd=p=(const BYTE*)(size_t)16; } + if (p==NULL) + { + len=0; + bEnd=p=(const BYTE*)(size_t)16; + } #endif if (len>=16) @@ -252,11 +282,24 @@ FORCE_INLINE U32 XXH32_endian_align(const void* input, unsigned int len, U32 see do { - v1 += XXH_get32bits(p) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4; - v2 += XXH_get32bits(p) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4; - v3 += XXH_get32bits(p) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4; - v4 += XXH_get32bits(p) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4; - } while (p<=limit); + v1 += XXH_get32bits(p) * PRIME32_2; + v1 = XXH_rotl32(v1, 13); + v1 *= PRIME32_1; + p+=4; + v2 += XXH_get32bits(p) * PRIME32_2; + v2 = XXH_rotl32(v2, 13); + v2 *= PRIME32_1; + p+=4; + v3 += XXH_get32bits(p) * PRIME32_2; + v3 = XXH_rotl32(v3, 13); + v3 *= PRIME32_1; + p+=4; + v4 += XXH_get32bits(p) * PRIME32_2; + v4 = XXH_rotl32(v4, 13); + v4 *= PRIME32_1; + p+=4; + } + while (p<=limit); h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); } @@ -291,13 +334,14 @@ FORCE_INLINE U32 XXH32_endian_align(const void* input, unsigned int len, U32 see } -U32 XXH32(const void* input, unsigned int len, U32 seed) +unsigned int XXH32 (const void* input, size_t len, unsigned seed) { #if 0 // Simple version, good for code maintenance, but unfortunately slow for small inputs - void* state = XXH32_init(seed); - XXH32_update(state, input, len); - return XXH32_digest(state); + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, input, len); + return XXH32_digest(&state); #else XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; @@ -318,7 +362,7 @@ U32 XXH32(const void* input, unsigned int len, U32 seed) #endif } -FORCE_INLINE U64 XXH64_endian_align(const void* input, unsigned int len, U64 seed, XXH_endianess endian, XXH_alignment align) +FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) { const BYTE* p = (const BYTE*)input; const BYTE* bEnd = p + len; @@ -326,7 +370,11 @@ FORCE_INLINE U64 XXH64_endian_align(const void* input, unsigned int len, U64 see #define XXH_get64bits(p) XXH_readLE64_align((const U64*)p, endian, align) #ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) { len=0; bEnd=p=(const BYTE*)(size_t)32; } + if (p==NULL) + { + len=0; + bEnd=p=(const BYTE*)(size_t)32; + } #endif if (len>=32) @@ -339,25 +387,50 @@ FORCE_INLINE U64 XXH64_endian_align(const void* input, unsigned int len, U64 see do { - v1 += XXH_get64bits(p) * PRIME64_2; p+=8; v1 = XXH_rotl64(v1, 31); v1 *= PRIME64_1; - v2 += XXH_get64bits(p) * PRIME64_2; p+=8; v2 = XXH_rotl64(v2, 31); v2 *= PRIME64_1; - v3 += XXH_get64bits(p) * PRIME64_2; p+=8; v3 = XXH_rotl64(v3, 31); v3 *= PRIME64_1; - v4 += XXH_get64bits(p) * PRIME64_2; p+=8; v4 = XXH_rotl64(v4, 31); v4 *= PRIME64_1; - } while (p<=limit); + v1 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + v2 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + v3 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + v4 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + } + while (p<=limit); h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - v1 *= PRIME64_2; v1 = XXH_rotl64(v1, 31); v1 *= PRIME64_1; h64 ^= v1; - h64 = h64 * PRIME64_1 + PRIME64_4; - - v2 *= PRIME64_2; v2 = XXH_rotl64(v2, 31); v2 *= PRIME64_1; h64 ^= v2; - h64 = h64 * PRIME64_1 + PRIME64_4; - - v3 *= PRIME64_2; v3 = XXH_rotl64(v3, 31); v3 *= PRIME64_1; h64 ^= v3; - h64 = h64 * PRIME64_1 + PRIME64_4; - - v4 *= PRIME64_2; v4 = XXH_rotl64(v4, 31); v4 *= PRIME64_1; h64 ^= v4; - h64 = h64 * PRIME64_1 + PRIME64_4; + v1 *= PRIME64_2; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + h64 ^= v1; + h64 = h64 * PRIME64_1 + PRIME64_4; + + v2 *= PRIME64_2; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + h64 ^= v2; + h64 = h64 * PRIME64_1 + PRIME64_4; + + v3 *= PRIME64_2; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + h64 ^= v3; + h64 = h64 * PRIME64_1 + PRIME64_4; + + v4 *= PRIME64_2; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + h64 ^= v4; + h64 = h64 * PRIME64_1 + PRIME64_4; } else { @@ -368,22 +441,25 @@ FORCE_INLINE U64 XXH64_endian_align(const void* input, unsigned int len, U64 see while (p+8<=bEnd) { - U64 k1 = XXH_get64bits(p); - k1 *= PRIME64_2; k1 = XXH_rotl64(k1,31); k1 *= PRIME64_1; h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; + U64 k1 = XXH_get64bits(p); + k1 *= PRIME64_2; + k1 = XXH_rotl64(k1,31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; } if (p+4<=bEnd) { - h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; + h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; p+=4; } while (p= sizeof(struct XXH_state32_t)); // A compilation error here means XXH32_SIZEOFSTATE is not large enough - return sizeof(struct XXH_state32_t); + XXH_STATIC_ASSERT(sizeof(XXH32_state_t) >= sizeof(XXH_istate32_t)); // A compilation error here means XXH32_state_t is not large enough + return (XXH32_state_t*)malloc(sizeof(XXH32_state_t)); } +XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + free(statePtr); + return XXH_OK; +}; -int XXH64_sizeofState(void) +XXH64_state_t* XXH64_createState(void) { - XXH_STATIC_ASSERT(XXH64_SIZEOFSTATE >= sizeof(struct XXH_state64_t)); // A compilation error here means XXH64_SIZEOFSTATE is not large enough - return sizeof(struct XXH_state64_t); + XXH_STATIC_ASSERT(sizeof(XXH64_state_t) >= sizeof(XXH_istate64_t)); // A compilation error here means XXH64_state_t is not large enough + return (XXH64_state_t*)malloc(sizeof(XXH64_state_t)); } +XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + free(statePtr); + return XXH_OK; +}; + +/*** Hash feed ***/ -XXH_errorcode XXH32_resetState(void* state_in, U32 seed) +XXH_errorcode XXH32_reset(XXH32_state_t* state_in, U32 seed) { - struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; + XXH_istate32_t* state = (XXH_istate32_t*) state_in; state->seed = seed; state->v1 = seed + PRIME32_1 + PRIME32_2; state->v2 = seed + PRIME32_2; @@ -473,9 +570,9 @@ XXH_errorcode XXH32_resetState(void* state_in, U32 seed) return XXH_OK; } -XXH_errorcode XXH64_resetState(void* state_in, unsigned long long seed) +XXH_errorcode XXH64_reset(XXH64_state_t* state_in, unsigned long long seed) { - struct XXH_state64_t * state = (struct XXH_state64_t *) state_in; + XXH_istate64_t* state = (XXH_istate64_t*) state_in; state->seed = seed; state->v1 = seed + PRIME64_1 + PRIME64_2; state->v2 = seed + PRIME64_2; @@ -487,24 +584,9 @@ XXH_errorcode XXH64_resetState(void* state_in, unsigned long long seed) } -void* XXH32_init (U32 seed) +FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state_in, const void* input, size_t len, XXH_endianess endian) { - void* state = XXH_malloc (sizeof(struct XXH_state32_t)); - if (state != NULL) XXH32_resetState(state, seed); - return state; -} - -void* XXH64_init (unsigned long long seed) -{ - void* state = XXH_malloc (sizeof(struct XXH_state64_t)); - if (state != NULL) XXH64_resetState(state, seed); - return state; -} - - -FORCE_INLINE XXH_errorcode XXH32_update_endian (void* state_in, const void* input, int len, XXH_endianess endian) -{ - struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; + XXH_istate32_t* state = (XXH_istate32_t *) state_in; const BYTE* p = (const BYTE*)input; const BYTE* const bEnd = p + len; @@ -517,7 +599,7 @@ FORCE_INLINE XXH_errorcode XXH32_update_endian (void* state_in, const void* inpu if (state->memsize + len < 16) // fill in tmp buffer { XXH_memcpy(state->memory + state->memsize, input, len); - state->memsize += len; + state->memsize += (U32)len; return XXH_OK; } @@ -526,10 +608,22 @@ FORCE_INLINE XXH_errorcode XXH32_update_endian (void* state_in, const void* inpu XXH_memcpy(state->memory + state->memsize, input, 16-state->memsize); { const U32* p32 = (const U32*)state->memory; - state->v1 += XXH_readLE32(p32, endian) * PRIME32_2; state->v1 = XXH_rotl32(state->v1, 13); state->v1 *= PRIME32_1; p32++; - state->v2 += XXH_readLE32(p32, endian) * PRIME32_2; state->v2 = XXH_rotl32(state->v2, 13); state->v2 *= PRIME32_1; p32++; - state->v3 += XXH_readLE32(p32, endian) * PRIME32_2; state->v3 = XXH_rotl32(state->v3, 13); state->v3 *= PRIME32_1; p32++; - state->v4 += XXH_readLE32(p32, endian) * PRIME32_2; state->v4 = XXH_rotl32(state->v4, 13); state->v4 *= PRIME32_1; p32++; + state->v1 += XXH_readLE32(p32, endian) * PRIME32_2; + state->v1 = XXH_rotl32(state->v1, 13); + state->v1 *= PRIME32_1; + p32++; + state->v2 += XXH_readLE32(p32, endian) * PRIME32_2; + state->v2 = XXH_rotl32(state->v2, 13); + state->v2 *= PRIME32_1; + p32++; + state->v3 += XXH_readLE32(p32, endian) * PRIME32_2; + state->v3 = XXH_rotl32(state->v3, 13); + state->v3 *= PRIME32_1; + p32++; + state->v4 += XXH_readLE32(p32, endian) * PRIME32_2; + state->v4 = XXH_rotl32(state->v4, 13); + state->v4 *= PRIME32_1; + p32++; } p += 16-state->memsize; state->memsize = 0; @@ -545,11 +639,24 @@ FORCE_INLINE XXH_errorcode XXH32_update_endian (void* state_in, const void* inpu do { - v1 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4; - v2 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4; - v3 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4; - v4 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4; - } while (p<=limit); + v1 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; + v1 = XXH_rotl32(v1, 13); + v1 *= PRIME32_1; + p+=4; + v2 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; + v2 = XXH_rotl32(v2, 13); + v2 *= PRIME32_1; + p+=4; + v3 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; + v3 = XXH_rotl32(v3, 13); + v3 *= PRIME32_1; + p+=4; + v4 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; + v4 = XXH_rotl32(v4, 13); + v4 *= PRIME32_1; + p+=4; + } + while (p<=limit); state->v1 = v1; state->v2 = v2; @@ -566,7 +673,7 @@ FORCE_INLINE XXH_errorcode XXH32_update_endian (void* state_in, const void* inpu return XXH_OK; } -XXH_errorcode XXH32_update (void* state_in, const void* input, unsigned int len) +XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) { XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; @@ -578,9 +685,9 @@ XXH_errorcode XXH32_update (void* state_in, const void* input, unsigned int len) -FORCE_INLINE U32 XXH32_intermediateDigest_endian (void* state_in, XXH_endianess endian) +FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endianess endian) { - struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; + XXH_istate32_t* state = (XXH_istate32_t*) state_in; const BYTE * p = (const BYTE*)state->memory; BYTE* bEnd = (BYTE*)state->memory + state->memsize; U32 h32; @@ -620,30 +727,20 @@ FORCE_INLINE U32 XXH32_intermediateDigest_endian (void* state_in, XXH_endianess } -U32 XXH32_intermediateDigest (void* state_in) +U32 XXH32_digest (const XXH32_state_t* state_in) { XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_intermediateDigest_endian(state_in, XXH_littleEndian); + return XXH32_digest_endian(state_in, XXH_littleEndian); else - return XXH32_intermediateDigest_endian(state_in, XXH_bigEndian); -} - - -U32 XXH32_digest (void* state_in) -{ - U32 h32 = XXH32_intermediateDigest(state_in); - - XXH_free(state_in); - - return h32; + return XXH32_digest_endian(state_in, XXH_bigEndian); } -FORCE_INLINE XXH_errorcode XXH64_update_endian (void* state_in, const void* input, int len, XXH_endianess endian) +FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state_in, const void* input, size_t len, XXH_endianess endian) { - struct XXH_state64_t * state = (struct XXH_state64_t *) state_in; + XXH_istate64_t * state = (XXH_istate64_t *) state_in; const BYTE* p = (const BYTE*)input; const BYTE* const bEnd = p + len; @@ -656,7 +753,7 @@ FORCE_INLINE XXH_errorcode XXH64_update_endian (void* state_in, const void* inpu if (state->memsize + len < 32) // fill in tmp buffer { XXH_memcpy(state->memory + state->memsize, input, len); - state->memsize += len; + state->memsize += (U32)len; return XXH_OK; } @@ -665,10 +762,22 @@ FORCE_INLINE XXH_errorcode XXH64_update_endian (void* state_in, const void* inpu XXH_memcpy(state->memory + state->memsize, input, 32-state->memsize); { const U64* p64 = (const U64*)state->memory; - state->v1 += XXH_readLE64(p64, endian) * PRIME64_2; state->v1 = XXH_rotl64(state->v1, 31); state->v1 *= PRIME64_1; p64++; - state->v2 += XXH_readLE64(p64, endian) * PRIME64_2; state->v2 = XXH_rotl64(state->v2, 31); state->v2 *= PRIME64_1; p64++; - state->v3 += XXH_readLE64(p64, endian) * PRIME64_2; state->v3 = XXH_rotl64(state->v3, 31); state->v3 *= PRIME64_1; p64++; - state->v4 += XXH_readLE64(p64, endian) * PRIME64_2; state->v4 = XXH_rotl64(state->v4, 31); state->v4 *= PRIME64_1; p64++; + state->v1 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v1 = XXH_rotl64(state->v1, 31); + state->v1 *= PRIME64_1; + p64++; + state->v2 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v2 = XXH_rotl64(state->v2, 31); + state->v2 *= PRIME64_1; + p64++; + state->v3 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v3 = XXH_rotl64(state->v3, 31); + state->v3 *= PRIME64_1; + p64++; + state->v4 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v4 = XXH_rotl64(state->v4, 31); + state->v4 *= PRIME64_1; + p64++; } p += 32-state->memsize; state->memsize = 0; @@ -684,11 +793,24 @@ FORCE_INLINE XXH_errorcode XXH64_update_endian (void* state_in, const void* inpu do { - v1 += XXH_readLE64((const U64*)p, endian) * PRIME64_2; v1 = XXH_rotl64(v1, 31); v1 *= PRIME64_1; p+=8; - v2 += XXH_readLE64((const U64*)p, endian) * PRIME64_2; v2 = XXH_rotl64(v2, 31); v2 *= PRIME64_1; p+=8; - v3 += XXH_readLE64((const U64*)p, endian) * PRIME64_2; v3 = XXH_rotl64(v3, 31); v3 *= PRIME64_1; p+=8; - v4 += XXH_readLE64((const U64*)p, endian) * PRIME64_2; v4 = XXH_rotl64(v4, 31); v4 *= PRIME64_1; p+=8; - } while (p<=limit); + v1 += XXH_readLE64((const U64*)p, endian) * PRIME64_2; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + p+=8; + v2 += XXH_readLE64((const U64*)p, endian) * PRIME64_2; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + p+=8; + v3 += XXH_readLE64((const U64*)p, endian) * PRIME64_2; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + p+=8; + v4 += XXH_readLE64((const U64*)p, endian) * PRIME64_2; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + p+=8; + } + while (p<=limit); state->v1 = v1; state->v2 = v2; @@ -705,7 +827,7 @@ FORCE_INLINE XXH_errorcode XXH64_update_endian (void* state_in, const void* inpu return XXH_OK; } -XXH_errorcode XXH64_update (void* state_in, const void* input, unsigned int len) +XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) { XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; @@ -717,33 +839,45 @@ XXH_errorcode XXH64_update (void* state_in, const void* input, unsigned int len) -FORCE_INLINE U64 XXH64_intermediateDigest_endian (void* state_in, XXH_endianess endian) +FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endianess endian) { - struct XXH_state64_t * state = (struct XXH_state64_t *) state_in; + XXH_istate64_t * state = (XXH_istate64_t *) state_in; const BYTE * p = (const BYTE*)state->memory; BYTE* bEnd = (BYTE*)state->memory + state->memsize; U64 h64; if (state->total_len >= 32) { - U64 v1 = state->v1; - U64 v2 = state->v2; - U64 v3 = state->v3; - U64 v4 = state->v4; + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - v1 *= PRIME64_2; v1 = XXH_rotl64(v1, 31); v1 *= PRIME64_1; h64 ^= v1; - h64 = h64*PRIME64_1 + PRIME64_4; - - v2 *= PRIME64_2; v2 = XXH_rotl64(v2, 31); v2 *= PRIME64_1; h64 ^= v2; - h64 = h64*PRIME64_1 + PRIME64_4; - - v3 *= PRIME64_2; v3 = XXH_rotl64(v3, 31); v3 *= PRIME64_1; h64 ^= v3; - h64 = h64*PRIME64_1 + PRIME64_4; - - v4 *= PRIME64_2; v4 = XXH_rotl64(v4, 31); v4 *= PRIME64_1; h64 ^= v4; - h64 = h64*PRIME64_1 + PRIME64_4; + v1 *= PRIME64_2; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + h64 ^= v1; + h64 = h64*PRIME64_1 + PRIME64_4; + + v2 *= PRIME64_2; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + h64 ^= v2; + h64 = h64*PRIME64_1 + PRIME64_4; + + v3 *= PRIME64_2; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + h64 ^= v3; + h64 = h64*PRIME64_1 + PRIME64_4; + + v4 *= PRIME64_2; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + h64 ^= v4; + h64 = h64*PRIME64_1 + PRIME64_4; } else { @@ -754,22 +888,25 @@ FORCE_INLINE U64 XXH64_intermediateDigest_endian (void* state_in, XXH_endianess while (p+8<=bEnd) { - U64 k1 = XXH_readLE64((const U64*)p, endian); - k1 *= PRIME64_2; k1 = XXH_rotl64(k1,31); k1 *= PRIME64_1; h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; + U64 k1 = XXH_readLE64((const U64*)p, endian); + k1 *= PRIME64_2; + k1 = XXH_rotl64(k1,31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; } if (p+4<=bEnd) { - h64 ^= (U64)(XXH_readLE32((const U32*)p, endian)) * PRIME64_1; + h64 ^= (U64)(XXH_readLE32((const U32*)p, endian)) * PRIME64_1; h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; p+=4; } while (p /* size_t */ + + +/***************************** Type *****************************/ typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; @@ -75,18 +81,16 @@ typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; Simple Hash Functions *****************************/ -unsigned int XXH32 (const void* input, unsigned int len, unsigned int seed); -unsigned long long XXH64 (const void* input, unsigned int len, unsigned long long seed); +unsigned int XXH32 (const void* input, size_t length, unsigned seed); +unsigned long long XXH64 (const void* input, size_t length, unsigned long long seed); /* XXH32() : - Calculate the 32-bits hash of sequence of length "len" stored at memory address "input". - The memory between input & input+len must be valid (allocated and read-accessible). + Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". + The memory between input & input+length must be valid (allocated and read-accessible). "seed" can be used to alter the result predictably. This function successfully passes all SMHasher tests. Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s - Note that "len" is type "int", which means it is limited to 2^31-1. - If your data is larger, use the advanced functions below. XXH64() : Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". */ @@ -96,70 +100,54 @@ XXH64() : /***************************** Advanced Hash Functions *****************************/ - -void* XXH32_init (unsigned int seed); -XXH_errorcode XXH32_update (void* state, const void* input, unsigned int len); -unsigned int XXH32_digest (void* state); - -void* XXH64_init (unsigned long long seed); -XXH_errorcode XXH64_update (void* state, const void* input, unsigned int len); -unsigned long long XXH64_digest (void* state); +typedef struct { long long ll[ 6]; } XXH32_state_t; +typedef struct { long long ll[11]; } XXH64_state_t; /* -These functions calculate the xxhash of an input provided in several small packets, -as opposed to an input provided as a single block. - -It must be started with : -void* XXHnn_init() -The function returns a pointer which holds the state of calculation. -If the pointer is NULL, allocation has failed, so no state can be tracked. - -The state pointer must be provided as "void* state" parameter for XXHnn_update(). -XXHnn_update() can be called as many times as necessary. -The user must provide a valid (allocated) input. -The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. -Note that "len" is type "int", which means it is limited to 2^31-1. -If your data is larger, it is recommended to chunk your data into blocks -of size for example 2^30 (1GB) to avoid any "int" overflow issue. +These structures allow static allocation of XXH states. +States must then be initialized using XXHnn_reset() before first use. -Finally, you can end the calculation anytime, by using XXHnn_digest(). -This function returns the final nn-bits hash. -You must provide the same "void* state" parameter created by XXHnn_init(). -Memory will be freed by XXHnn_digest(). +If you prefer dynamic allocation, please refer to functions below. */ +XXH32_state_t* XXH32_createState(void); +XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); -int XXH32_sizeofState(void); -XXH_errorcode XXH32_resetState(void* state, unsigned int seed); +XXH64_state_t* XXH64_createState(void); +XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); -#define XXH32_SIZEOFSTATE 48 -typedef struct { long long ll[(XXH32_SIZEOFSTATE+(sizeof(long long)-1))/sizeof(long long)]; } XXH32_stateSpace_t; +/* +These functions create and release memory for XXH state. +States must then be initialized using XXHnn_reset() before first use. +*/ -int XXH64_sizeofState(void); -XXH_errorcode XXH64_resetState(void* state, unsigned long long seed); -#define XXH64_SIZEOFSTATE 88 -typedef struct { long long ll[(XXH64_SIZEOFSTATE+(sizeof(long long)-1))/sizeof(long long)]; } XXH64_stateSpace_t; +XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned seed); +XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); +unsigned int XXH32_digest (const XXH32_state_t* statePtr); + +XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); +XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); +unsigned long long XXH64_digest (const XXH64_state_t* statePtr); /* -These functions allow user application to make its own allocation for state. +These functions calculate the xxHash of an input provided in multiple smaller packets, +as opposed to an input provided as a single block. -XXHnn_sizeofState() is used to know how much space must be allocated for the xxHash nn-bits state. -Note that the state must be aligned to access 'long long' fields. Memory must be allocated and referenced by a pointer. -This pointer must then be provided as 'state' into XXHnn_resetState(), which initializes the state. +XXH state space must first be allocated, using either static or dynamic method provided above. -For static allocation purposes (such as allocation on stack, or freestanding systems without malloc()), -use the structure XXHnn_stateSpace_t, which will ensure that memory space is large enough and correctly aligned to access 'long long' fields. -*/ +Start a new hash by initializing state with a seed, using XXHnn_reset(). +Then, feed the hash state by calling XXHnn_update() as many times as necessary. +Obviously, input must be valid, meaning allocated and read accessible. +The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. -unsigned int XXH32_intermediateDigest (void* state); -unsigned long long XXH64_intermediateDigest (void* state); -/* -These functions do the same as XXHnn_digest(), generating a nn-bit hash, -but preserve memory context. -This way, it becomes possible to generate intermediate hashes, and then continue feeding data with XXHnn_update(). -To free memory context, use XXHnn_digest(), or free(). +Finally, you can produce a hash anytime, by using XXHnn_digest(). +This function returns the final nn-bits hash. +You can nonetheless continue feeding the hash state with more input, +and therefore get some new hashes, by calling again XXHnn_digest(). + +When you are done, don't forget to free XXH state space, using typically XXHnn_freeState(). */ -- cgit v0.12 From 92574da1a020d68bf9204d1e452451b0d4922ed0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 9 Oct 2014 22:15:57 +0100 Subject: modified : fuzzer automatically selects seed --- programs/Makefile | 4 ++-- programs/fuzzer.c | 67 +++++++++++++++++++++++-------------------------------- 2 files changed, 30 insertions(+), 41 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index c653ec6..04694f2 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -164,10 +164,10 @@ test-fullbench32: fullbench32 ./fullbench32 --no-prompt $(BENCH_NB) $(TEST_FILES) test-fuzzer: fuzzer - ./fuzzer --no-prompt + ./fuzzer test-fuzzer32: fuzzer32 - ./fuzzer32 --no-prompt + ./fuzzer32 test-frame: frametest ./frametest diff --git a/programs/fuzzer.c b/programs/fuzzer.c index c98333d..22b934b 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -91,6 +91,8 @@ *****************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static const U32 refreshRate = 250; +static U32 g_time = 0; /***************************************** @@ -104,22 +106,23 @@ static int displayLevel = 2; /********************************************************* Fuzzer functions *********************************************************/ -static int FUZ_GetMilliStart(void) +static U32 FUZ_GetMilliStart(void) { - struct timeb tb; - int nCount; - ftime( &tb ); - nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000); - return nCount; + struct timeb tb; + U32 nCount; + ftime( &tb ); + nCount = (U32) (((tb.time & 0xFFFFF) * 1000) + tb.millitm); + return nCount; } -static int FUZ_GetMilliSpan( int nTimeStart ) +static U32 FUZ_GetMilliSpan(U32 nTimeStart) { - int nSpan = FUZ_GetMilliStart() - nTimeStart; - if ( nSpan < 0 ) - nSpan += 0x100000 * 1000; - return nSpan; + U32 nCurrent = FUZ_GetMilliStart(); + U32 nSpan = nCurrent - nTimeStart; + if (nTimeStart > nCurrent) + nSpan += 0x100000 * 1000; + return nSpan; } @@ -263,6 +266,16 @@ _overflowError: } +void FUZ_displayUpdate(int testNb) +{ + if ((FUZ_GetMilliSpan(g_time) > refreshRate) || (displayLevel>=3)) + { + g_time = FUZ_GetMilliStart(); + DISPLAY("\r%5u ", testNb); + if (displayLevel>=3) fflush(stdout); + } +} + #define FUZ_MAX(a,b) (a>b?a:b) @@ -285,7 +298,6 @@ int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) { void* LZ4continue; LZ4_stream_t LZ4dict; U32 crcOrig, crcCheck; - int displayRefresh; // init @@ -297,15 +309,6 @@ int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) { compressedBuffer = malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE)); decodedBuffer = malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE); - // display refresh rate - switch(displayLevel) - { - case 0: displayRefresh = nbCycles+1; break; - case 1: displayRefresh = FUZ_MAX(1, nbCycles / 100); break; - case 2: displayRefresh = 89; break; - default : displayRefresh=1; - } - // move to startCycle for (cycleNb = 0; cycleNb < startCycle; cycleNb++) { @@ -337,11 +340,7 @@ int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) { int dictSize, blockSize, blockStart, compressedSize, HCcompressedSize; int blockContinueCompressedSize; - if ((cycleNb%displayRefresh) == 0) - { - printf("\r%7i /%7i - ", cycleNb, nbCycles); - fflush(stdout); - } + FUZ_displayUpdate(cycleNb); // Select block to test blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE; @@ -641,8 +640,8 @@ int FUZ_usage(void) } -int main(int argc, char** argv) { - U32 timestamp = FUZ_GetMilliStart(); +int main(int argc, char** argv) +{ U32 seed=0; int seedset=0; int argNb; @@ -725,17 +724,7 @@ int main(int argc, char** argv) { // Get Seed printf("Starting LZ4 fuzzer (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION); - if (!seedset) - { - char userInput[50] = {0}; - printf("Select an Initialisation number (default : random) : "); - fflush(stdout); - if ( no_prompt || fgets(userInput, sizeof userInput, stdin) ) - { - if ( sscanf(userInput, "%u", &seed) == 1 ) {} - else seed = FUZ_GetMilliSpan(timestamp); - } - } + if (!seedset) seed = FUZ_GetMilliStart() % 10000; printf("Seed = %u\n", seed); if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba); -- cgit v0.12 From 4da47a2c67e0887e624928a37a109b5c3f90fd18 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 10 Oct 2014 20:58:42 +0100 Subject: fuzzer : pause at the end is disabled by default --- programs/fuzzer.c | 692 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 358 insertions(+), 334 deletions(-) diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 22b934b..283352c 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -98,7 +98,6 @@ static U32 g_time = 0; /***************************************** Local Parameters *****************************************/ -static int no_prompt = 0; static char* programName; static int displayLevel = 2; @@ -179,7 +178,7 @@ void FUZ_fillCompressibleNoiseBuffer(void* buffer, int bufferSize, double proba, #define MAX_NB_BUFF_I134 150 #define BLOCKSIZE_I134 (32 MB) -int FUZ_AddressOverflow(void) +static int FUZ_AddressOverflow(void) { char* buffers[MAX_NB_BUFF_I134+1] = {0}; int i, nbBuff=0; @@ -266,7 +265,7 @@ _overflowError: } -void FUZ_displayUpdate(int testNb) +static void FUZ_displayUpdate(int testNb) { if ((FUZ_GetMilliSpan(g_time) > refreshRate) || (displayLevel>=3)) { @@ -279,348 +278,353 @@ void FUZ_displayUpdate(int testNb) #define FUZ_MAX(a,b) (a>b?a:b) -int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) { - unsigned long long bytes = 0; - unsigned long long cbytes = 0; - unsigned long long hcbytes = 0; - unsigned long long ccbytes = 0; - void* CNBuffer; - char* compressedBuffer; - char* decodedBuffer; +static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) +{ + unsigned long long bytes = 0; + unsigned long long cbytes = 0; + unsigned long long hcbytes = 0; + unsigned long long ccbytes = 0; + void* CNBuffer; + char* compressedBuffer; + char* decodedBuffer; # define FUZ_max LZ4_COMPRESSBOUND(LEN) - unsigned int randState=seed; - int ret, cycleNb; + unsigned int randState=seed; + int ret, cycleNb; # define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %i : ", testNb); printf(__VA_ARGS__); \ - printf(" (seed %u, cycle %i) \n", seed, cycleNb); goto _output_error; } -# define FUZ_DISPLAYTEST { testNb++; ((displayLevel<3) || no_prompt) ? 0 : printf("%2i\b\b", testNb); if (displayLevel==4) fflush(stdout); } - void* stateLZ4 = malloc(LZ4_sizeofState()); - void* stateLZ4HC = malloc(LZ4_sizeofStateHC()); - void* LZ4continue; - LZ4_stream_t LZ4dict; - U32 crcOrig, crcCheck; - - - // init - memset(&LZ4dict, 0, sizeof(LZ4dict)); - - // Create compressible test buffer - CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); - FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); - compressedBuffer = malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE)); - decodedBuffer = malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE); - - // move to startCycle - for (cycleNb = 0; cycleNb < startCycle; cycleNb++) + printf(" (seed %u, cycle %i) \n", seed, cycleNb); goto _output_error; } +# define FUZ_DISPLAYTEST { testNb++; displayLevel<3 ? 0 : printf("%2i\b\b", testNb); if (displayLevel==4) fflush(stdout); } + void* stateLZ4 = malloc(LZ4_sizeofState()); + void* stateLZ4HC = malloc(LZ4_sizeofStateHC()); + void* LZ4continue; + LZ4_stream_t LZ4dict; + U32 crcOrig, crcCheck; + + + // init + memset(&LZ4dict, 0, sizeof(LZ4dict)); + + // Create compressible test buffer + CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); + FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); + compressedBuffer = malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE)); + decodedBuffer = malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE); + + // move to startCycle + for (cycleNb = 0; cycleNb < startCycle; cycleNb++) + { + // synd rand & dict + int dictSize, blockSize, blockStart; + char* dict; + char* block; + + blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE; + blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); + dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; + if (dictSize > blockStart) dictSize = blockStart; + block = ((char*)CNBuffer) + blockStart; + dict = block - dictSize; + LZ4_loadDict(&LZ4dict, dict, dictSize); + LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); + LZ4_loadDict(&LZ4dict, dict, dictSize); + LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); + LZ4_loadDict(&LZ4dict, dict, dictSize); + LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); + } + + // Test loop + for (cycleNb = startCycle; cycleNb < nbCycles; cycleNb++) + { + int testNb = 0; + char* dict; + char* block; + int dictSize, blockSize, blockStart, compressedSize, HCcompressedSize; + int blockContinueCompressedSize; + + FUZ_displayUpdate(cycleNb); + + // Select block to test + blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE; + blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); + dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; + if (dictSize > blockStart) dictSize = blockStart; + block = ((char*)CNBuffer) + blockStart; + dict = block - dictSize; + + /* Compression tests */ + + // Test compression HC + FUZ_DISPLAYTEST; + ret = LZ4_compressHC(block, compressedBuffer, blockSize); + FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed"); + HCcompressedSize = ret; + + // Test compression HC using external state + FUZ_DISPLAYTEST; + ret = LZ4_compressHC_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize); + FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed"); + + // Test compression using external state + FUZ_DISPLAYTEST; + ret = LZ4_compress_withState(stateLZ4, block, compressedBuffer, blockSize); + FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed"); + + // Test compression + FUZ_DISPLAYTEST; + ret = LZ4_compress(block, compressedBuffer, blockSize); + FUZ_CHECKTEST(ret==0, "LZ4_compress() failed"); + compressedSize = ret; + + /* Decompression tests */ + + crcOrig = XXH32(block, blockSize, 0); + + // Test decoding with output size being exactly what's necessary => must work + FUZ_DISPLAYTEST; + ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize); + FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space"); + FUZ_CHECKTEST(ret!=compressedSize, "LZ4_decompress_fast failed : did not fully read compressed data"); + crcCheck = XXH32(decodedBuffer, blockSize, 0); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data"); + + // Test decoding with one byte missing => must fail + FUZ_DISPLAYTEST; + decodedBuffer[blockSize-1] = 0; + ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1); + FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small"); + FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer"); + + // Test decoding with one byte too much => must fail + FUZ_DISPLAYTEST; + ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1); + FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large"); + + // Test decoding with output size exactly what's necessary => must work + FUZ_DISPLAYTEST; + decodedBuffer[blockSize] = 0; + ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize); + FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space"); + FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data"); + FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size"); + crcCheck = XXH32(decodedBuffer, blockSize, 0); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); + + // Test decoding with more than enough output size => must work + FUZ_DISPLAYTEST; + decodedBuffer[blockSize] = 0; + decodedBuffer[blockSize+1] = 0; + ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize+1); + FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite amply sufficient space"); + FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data"); + //FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe wrote more than (unknown) target size"); // well, is that an issue ? + FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ4_decompress_safe overrun specified output buffer size"); + crcCheck = XXH32(decodedBuffer, blockSize, 0); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); + + // Test decoding with output size being one byte too short => must fail + FUZ_DISPLAYTEST; + decodedBuffer[blockSize-1] = 0; + ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-1); + FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short"); + FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe overrun specified output buffer size"); + + // Test decoding with output size being 10 bytes too short => must fail + FUZ_DISPLAYTEST; + if (blockSize>10) { - // synd rand & dict - int dictSize, blockSize, blockStart; - char* dict; - char* block; - - blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE; - blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); - dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; - if (dictSize > blockStart) dictSize = blockStart; - block = ((char*)CNBuffer) + blockStart; - dict = block - dictSize; - LZ4_loadDict(&LZ4dict, dict, dictSize); - LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); - LZ4_loadDict(&LZ4dict, dict, dictSize); - LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); - LZ4_loadDict(&LZ4dict, dict, dictSize); - LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); + decodedBuffer[blockSize-10] = 0; + ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-10); + FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being 10 bytes too short"); + FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe overrun specified output buffer size"); } - // Test loop - for (cycleNb = startCycle; cycleNb < nbCycles; cycleNb++) + // Test decoding with input size being one byte too short => must fail + FUZ_DISPLAYTEST; + ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize); + FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize); + + // Test decoding with input size being one byte too large => must fail + FUZ_DISPLAYTEST; + decodedBuffer[blockSize] = 0; + ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize+1, blockSize); + FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large"); + FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size"); + + // Test partial decoding with target output size being max/2 => must work + FUZ_DISPLAYTEST; + ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize/2, blockSize); + FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space"); + + // Test partial decoding with target output size being just below max => must work + FUZ_DISPLAYTEST; + ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize-3, blockSize); + FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space"); + + /* Test Compression with limited output size */ + + // Test compression with output size being exactly what's necessary (should work) + FUZ_DISPLAYTEST; + ret = LZ4_compress_limitedOutput(block, compressedBuffer, blockSize, compressedSize); + FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space"); + + // Test compression with output size being exactly what's necessary and external state (should work) + FUZ_DISPLAYTEST; + ret = LZ4_compress_limitedOutput_withState(stateLZ4, block, compressedBuffer, blockSize, compressedSize); + FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput_withState() failed despite sufficient space"); + + // Test HC compression with output size being exactly what's necessary (should work) + FUZ_DISPLAYTEST; + ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize); + FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space"); + + // Test HC compression with output size being exactly what's necessary (should work) + FUZ_DISPLAYTEST; + ret = LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize); + FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space"); + + // Test compression with just one missing byte into output buffer => must fail + FUZ_DISPLAYTEST; + compressedBuffer[compressedSize-1] = 0; + ret = LZ4_compress_limitedOutput(block, compressedBuffer, blockSize, compressedSize-1); + FUZ_CHECKTEST(ret, "LZ4_compress_limitedOutput should have failed (output buffer too small by 1 byte)"); + FUZ_CHECKTEST(compressedBuffer[compressedSize-1], "LZ4_compress_limitedOutput overran output buffer") + + // Test HC compression with just one missing byte into output buffer => must fail + FUZ_DISPLAYTEST; + compressedBuffer[compressedSize-1] = 0; + ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize-1); + FUZ_CHECKTEST(ret, "LZ4_compressHC_limitedOutput should have failed (output buffer too small by 1 byte)"); + FUZ_CHECKTEST(compressedBuffer[compressedSize-1], "LZ4_compressHC_limitedOutput overran output buffer") + + /* Dictionary tests */ + + // Compress using dictionary + FUZ_DISPLAYTEST; + LZ4continue = LZ4_create (dict); + LZ4_compress_continue (LZ4continue, dict, compressedBuffer, dictSize); // Just to fill hash tables + blockContinueCompressedSize = LZ4_compress_continue (LZ4continue, block, compressedBuffer, blockSize); + FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed"); + free (LZ4continue); + + // Decompress with dictionary as prefix + FUZ_DISPLAYTEST; + memcpy(decodedBuffer, dict, dictSize); + ret = LZ4_decompress_fast_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockSize); + FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_withPrefix64k did not read all compressed block input"); + crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); + if (crcCheck!=crcOrig) { - int testNb = 0; - char* dict; - char* block; - int dictSize, blockSize, blockStart, compressedSize, HCcompressedSize; - int blockContinueCompressedSize; - - FUZ_displayUpdate(cycleNb); - - // Select block to test - blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE; - blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); - dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; - if (dictSize > blockStart) dictSize = blockStart; - block = ((char*)CNBuffer) + blockStart; - dict = block - dictSize; - - /* Compression tests */ - - // Test compression HC - FUZ_DISPLAYTEST; - ret = LZ4_compressHC(block, compressedBuffer, blockSize); - FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed"); - HCcompressedSize = ret; - - // Test compression HC using external state - FUZ_DISPLAYTEST; - ret = LZ4_compressHC_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize); - FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed"); - - // Test compression using external state - FUZ_DISPLAYTEST; - ret = LZ4_compress_withState(stateLZ4, block, compressedBuffer, blockSize); - FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed"); - - // Test compression - FUZ_DISPLAYTEST; - ret = LZ4_compress(block, compressedBuffer, blockSize); - FUZ_CHECKTEST(ret==0, "LZ4_compress() failed"); - compressedSize = ret; - - /* Decompression tests */ - - crcOrig = XXH32(block, blockSize, 0); - - // Test decoding with output size being exactly what's necessary => must work - FUZ_DISPLAYTEST; - ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space"); - FUZ_CHECKTEST(ret!=compressedSize, "LZ4_decompress_fast failed : did not fully read compressed data"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data"); - - // Test decoding with one byte missing => must fail - FUZ_DISPLAYTEST; - decodedBuffer[blockSize-1] = 0; - ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small"); - FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer"); - - // Test decoding with one byte too much => must fail - FUZ_DISPLAYTEST; - ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large"); - - // Test decoding with output size exactly what's necessary => must work - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space"); - FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data"); - FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); - - // Test decoding with more than enough output size => must work - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - decodedBuffer[blockSize+1] = 0; - ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize+1); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite amply sufficient space"); - FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data"); - //FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe wrote more than (unknown) target size"); // well, is that an issue ? - FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ4_decompress_safe overrun specified output buffer size"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); - - // Test decoding with output size being one byte too short => must fail - FUZ_DISPLAYTEST; - decodedBuffer[blockSize-1] = 0; - ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-1); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short"); - FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe overrun specified output buffer size"); - - // Test decoding with output size being 10 bytes too short => must fail - FUZ_DISPLAYTEST; - if (blockSize>10) - { - decodedBuffer[blockSize-10] = 0; - ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-10); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being 10 bytes too short"); - FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe overrun specified output buffer size"); - } - - // Test decoding with input size being one byte too short => must fail - FUZ_DISPLAYTEST; - ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize); - - // Test decoding with input size being one byte too large => must fail - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize+1, blockSize); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large"); - FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size"); - - // Test partial decoding with target output size being max/2 => must work - FUZ_DISPLAYTEST; - ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize/2, blockSize); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space"); - - // Test partial decoding with target output size being just below max => must work - FUZ_DISPLAYTEST; - ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize-3, blockSize); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space"); - - /* Test Compression with limited output size */ - - // Test compression with output size being exactly what's necessary (should work) - FUZ_DISPLAYTEST; - ret = LZ4_compress_limitedOutput(block, compressedBuffer, blockSize, compressedSize); - FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space"); - - // Test compression with output size being exactly what's necessary and external state (should work) - FUZ_DISPLAYTEST; - ret = LZ4_compress_limitedOutput_withState(stateLZ4, block, compressedBuffer, blockSize, compressedSize); - FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput_withState() failed despite sufficient space"); - - // Test HC compression with output size being exactly what's necessary (should work) - FUZ_DISPLAYTEST; - ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize); - FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space"); - - // Test HC compression with output size being exactly what's necessary (should work) - FUZ_DISPLAYTEST; - ret = LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize); - FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space"); - - // Test compression with just one missing byte into output buffer => must fail - FUZ_DISPLAYTEST; - compressedBuffer[compressedSize-1] = 0; - ret = LZ4_compress_limitedOutput(block, compressedBuffer, blockSize, compressedSize-1); - FUZ_CHECKTEST(ret, "LZ4_compress_limitedOutput should have failed (output buffer too small by 1 byte)"); - FUZ_CHECKTEST(compressedBuffer[compressedSize-1], "LZ4_compress_limitedOutput overran output buffer") - - // Test HC compression with just one missing byte into output buffer => must fail - FUZ_DISPLAYTEST; - compressedBuffer[compressedSize-1] = 0; - ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize-1); - FUZ_CHECKTEST(ret, "LZ4_compressHC_limitedOutput should have failed (output buffer too small by 1 byte)"); - FUZ_CHECKTEST(compressedBuffer[compressedSize-1], "LZ4_compressHC_limitedOutput overran output buffer") - - /* Dictionary tests */ - - // Compress using dictionary - FUZ_DISPLAYTEST; - LZ4continue = LZ4_create (dict); - LZ4_compress_continue (LZ4continue, dict, compressedBuffer, dictSize); // Just to fill hash tables - blockContinueCompressedSize = LZ4_compress_continue (LZ4continue, block, compressedBuffer, blockSize); - FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed"); - free (LZ4continue); - - // Decompress with dictionary as prefix - FUZ_DISPLAYTEST; - memcpy(decodedBuffer, dict, dictSize); - ret = LZ4_decompress_fast_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockSize); - FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_withPrefix64k did not read all compressed block input"); - crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); - if (crcCheck!=crcOrig) - { - int i=0; - while (block[i]==decodedBuffer[i]) i++; - printf("Wrong Byte at position %i/%i\n", i, blockSize); + int i=0; + while (block[i]==decodedBuffer[i]) i++; + printf("Wrong Byte at position %i/%i\n", i, blockSize); - } - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_withPrefix64k corrupted decoded data (dict %i)", dictSize); - - FUZ_DISPLAYTEST; - ret = LZ4_decompress_safe_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize); - FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_withPrefix64k did not regenerate original data"); - crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_withPrefix64k corrupted decoded data"); - - // Compress using External dictionary - FUZ_DISPLAYTEST; - dict -= 9; // Separation, so it is an ExtDict - if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; - LZ4_loadDict(&LZ4dict, dict, dictSize); - blockContinueCompressedSize = LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); - FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed"); - - FUZ_DISPLAYTEST; - LZ4_loadDict(&LZ4dict, dict, dictSize); - ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); - FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer"); - - FUZ_DISPLAYTEST; - LZ4_loadDict(&LZ4dict, dict, dictSize); - ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize); - FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); - FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer"); - - // Decompress with dictionary as external - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize); - FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input"); - FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size") - crcCheck = XXH32(decodedBuffer, blockSize, 0); - if (crcCheck!=crcOrig) - { - int i=0; - while (block[i]==decodedBuffer[i]) i++; - printf("Wrong Byte at position %i/%i\n", i, blockSize); + } + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_withPrefix64k corrupted decoded data (dict %i)", dictSize); + + FUZ_DISPLAYTEST; + ret = LZ4_decompress_safe_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize); + FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_withPrefix64k did not regenerate original data"); + crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_withPrefix64k corrupted decoded data"); + + // Compress using External dictionary + FUZ_DISPLAYTEST; + dict -= 9; // Separation, so it is an ExtDict + if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; + LZ4_loadDict(&LZ4dict, dict, dictSize); + blockContinueCompressedSize = LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); + FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed"); + + FUZ_DISPLAYTEST; + LZ4_loadDict(&LZ4dict, dict, dictSize); + ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); + FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer"); + + FUZ_DISPLAYTEST; + LZ4_loadDict(&LZ4dict, dict, dictSize); + ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize); + FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); + FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer"); + + // Decompress with dictionary as external + FUZ_DISPLAYTEST; + decodedBuffer[blockSize] = 0; + ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize); + FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input"); + FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size") + crcCheck = XXH32(decodedBuffer, blockSize, 0); + if (crcCheck!=crcOrig) + { + int i=0; + while (block[i]==decodedBuffer[i]) i++; + printf("Wrong Byte at position %i/%i\n", i, blockSize); - } - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); - - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); - FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); - FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size") - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); - - FUZ_DISPLAYTEST; - decodedBuffer[blockSize-1] = 0; - ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_withDict should have failed : wrong original size (-1 byte)"); - FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size"); - - FUZ_DISPLAYTEST; - decodedBuffer[blockSize-1] = 0; - ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)"); - FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); - - FUZ_DISPLAYTEST; - if (blockSize > 10) - { - decodedBuffer[blockSize-10] = 0; - ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-10, dict, dictSize); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-10 byte)"); - FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-10 byte) (blockSize=%i)", blockSize); - } + } + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); + + FUZ_DISPLAYTEST; + decodedBuffer[blockSize] = 0; + ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); + FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); + FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size") + crcCheck = XXH32(decodedBuffer, blockSize, 0); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); + + FUZ_DISPLAYTEST; + decodedBuffer[blockSize-1] = 0; + ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize); + FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_withDict should have failed : wrong original size (-1 byte)"); + FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size"); + + FUZ_DISPLAYTEST; + decodedBuffer[blockSize-1] = 0; + ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize); + FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)"); + FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); + + FUZ_DISPLAYTEST; + if (blockSize > 10) + { + decodedBuffer[blockSize-10] = 0; + ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-10, dict, dictSize); + FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-10 byte)"); + FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-10 byte) (blockSize=%i)", blockSize); + } - // Fill stats - bytes += blockSize; - cbytes += compressedSize; - hcbytes += HCcompressedSize; - ccbytes += blockContinueCompressedSize; - } + // Fill stats + bytes += blockSize; + cbytes += compressedSize; + hcbytes += HCcompressedSize; + ccbytes += blockContinueCompressedSize; + } - printf("\r%7i /%7i - ", cycleNb, nbCycles); - printf("all tests completed successfully \n"); - printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100); - printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100); - printf("ratio with dict: %0.3f%%\n", (double)ccbytes/bytes*100); + printf("\r%7i /%7i - ", cycleNb, nbCycles); + printf("all tests completed successfully \n"); + printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100); + printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100); + printf("ratio with dict: %0.3f%%\n", (double)ccbytes/bytes*100); - // unalloc - if(!no_prompt) getchar(); + // unalloc + { + int result = 0; +_exit: free(CNBuffer); free(compressedBuffer); free(decodedBuffer); free(stateLZ4); free(stateLZ4HC); - return 0; + return result; _output_error: - if(!no_prompt) getchar(); - free(CNBuffer); - free(compressedBuffer); - free(decodedBuffer); - free(stateLZ4); - free(stateLZ4HC); - return 1; + result = 1; + goto _exit; + } +} + + +static void FUZ_unitTests(void) +{ + FUZ_AddressOverflow(); } @@ -633,8 +637,9 @@ int FUZ_usage(void) DISPLAY( " -i# : Nb of tests (default:%i) \n", NB_ATTEMPTS); DISPLAY( " -s# : Select seed (default:prompt user)\n"); DISPLAY( " -t# : Select starting test number (default:0)\n"); - DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); + DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); DISPLAY( " -v : verbose\n"); + DISPLAY( " -p : pause at the end\n"); DISPLAY( " -h : display help and exit\n"); return 0; } @@ -648,6 +653,7 @@ int main(int argc, char** argv) int nbTests = NB_ATTEMPTS; int testNb = 0; int proba = FUZ_COMPRESSIBILITY_DEFAULT; + int pause=0; // Check command line programName = argv[0]; @@ -660,19 +666,26 @@ int main(int argc, char** argv) // Decode command (note : aggregated commands are allowed) if (argument[0]=='-') { - if (!strcmp(argument, "--no-prompt")) { no_prompt=1; seedset=1; displayLevel=1; continue; } + if (!strcmp(argument, "--no-prompt")) { pause=0; seedset=1; displayLevel=1; continue; } while (argument[1]!=0) { argument++; switch(*argument) { - case 'h': + case 'h': /* display help */ return FUZ_usage(); - case 'v': + + case 'v': /* verbose mode */ argument++; displayLevel=4; break; + + case 'p': /* pause at the end */ + argument++; + pause=1; + break; + case 'i': argument++; nbTests=0; @@ -683,6 +696,7 @@ int main(int argc, char** argv) argument++; } break; + case 's': argument++; seed=0; seedset=1; @@ -693,7 +707,8 @@ int main(int argc, char** argv) argument++; } break; - case 't': + + case 't': /* select starting test nb */ argument++; testNb=0; while ((*argument>='0') && (*argument<='9')) @@ -703,7 +718,8 @@ int main(int argc, char** argv) argument++; } break; - case 'p': + + case 'P': /* change probability */ argument++; proba=0; while ((*argument>='0') && (*argument<='9')) @@ -728,9 +744,17 @@ int main(int argc, char** argv) printf("Seed = %u\n", seed); if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba); - FUZ_AddressOverflow(); + FUZ_unitTests(); if (nbTests<=0) nbTests=1; - return FUZ_test(seed, nbTests, testNb, ((double)proba) / 100); + { + int result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100); + if (pause) + { + DISPLAY("press enter ... \n"); + getchar(); + } + return result; + } } -- cgit v0.12 From d239a23337e5eba41f557b48eb26f0db81b28f26 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 18 Oct 2014 11:18:14 +0100 Subject: updated LZ4HC API --- lz4.h | 10 +- lz4hc.c | 368 +++++++++++++++++++++++++-------------------------- lz4hc.h | 58 +++++++- programs/fullbench.c | 4 +- programs/fuzzer.c | 90 ++++++++++++- 5 files changed, 333 insertions(+), 197 deletions(-) diff --git a/lz4.h b/lz4.h index 44ada14..7ad736f 100644 --- a/lz4.h +++ b/lz4.h @@ -193,7 +193,7 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_streamPtr); * LZ4_freeStream releases its memory. */ LZ4_stream_t* LZ4_createStream(void); -int LZ4_freeStream (LZ4_stream_t* LZ4_stream); +int LZ4_freeStream (LZ4_stream_t* LZ4_streamPtr); /* * LZ4_loadDict @@ -202,21 +202,21 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_stream); * Loading a size of 0 is allowed. * Return : 1 if OK, 0 if error */ -int LZ4_loadDict (LZ4_stream_t* LZ4_stream, const char* dictionary, int dictSize); +int LZ4_loadDict (LZ4_stream_t* LZ4_streamPtr, const char* dictionary, int dictSize); /* * LZ4_compress_continue * Compress data block 'source', using blocks compressed before as dictionary to improve compression ratio * Previous data blocks are assumed to still be present at their previous location. */ -int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize); +int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); /* * LZ4_compress_limitedOutput_continue * Same as before, but also specify a maximum target compressed size (maxOutputSize) * If objective cannot be met, compression exits, and returns a zero. */ -int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize); +int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); /* * LZ4_saveDict @@ -227,7 +227,7 @@ int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* s * Return : dictionary size in bytes, or 0 if error * Note : any dictSize > 64 KB will be interpreted as 64KB. */ -int LZ4_saveDict (LZ4_stream_t* LZ4_stream, char* safeBuffer, int dictSize); +int LZ4_saveDict (LZ4_stream_t* LZ4_streamPtr, char* safeBuffer, int dictSize); /************************************************ diff --git a/lz4hc.c b/lz4hc.c index 34a6173..2cf494e 100644 --- a/lz4hc.c +++ b/lz4hc.c @@ -214,8 +214,8 @@ typedef struct _U64_S { U64 v; } _PACKED U64_S; #define MINLENGTH (MFLIMIT+1) #define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH) -#define KB *(1U<<10) -#define MB *(1U<<20) +#define KB *(1<<10) +#define MB *(1<<20) #define GB *(1U<<30) @@ -226,16 +226,12 @@ typedef struct _U64_S { U64 v; } _PACKED U64_S; # define STEPSIZE 8 # define LZ4_COPYSTEP(s,d) A64(d) = A64(s); d+=8; s+=8; # define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d) -# define AARCH A64 -# define HTYPE U32 -# define INITBASE(b,s) const BYTE* const b = s +# define AARCH A64 #else /* 32-bit */ # define STEPSIZE 4 # define LZ4_COPYSTEP(s,d) A32(d) = A32(s); d+=4; s+=4; # define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d); LZ4_COPYSTEP(s,d); -# define AARCH A32 -# define HTYPE U32 -# define INITBASE(b,s) const BYTE* const b = s +# define AARCH A32 #endif #if defined(LZ4_BIG_ENDIAN) @@ -252,18 +248,20 @@ typedef struct _U64_S { U64 v; } _PACKED U64_S; **************************************/ typedef struct { + U32 hashTable[HASHTABLESIZE]; + U16 chainTable[MAXD]; const BYTE* inputBuffer; const BYTE* base; const BYTE* end; - HTYPE hashTable[HASHTABLESIZE]; - U16 chainTable[MAXD]; - const BYTE* nextToUpdate; + U32 nextToUpdate; + U32 compressionLevel; } LZ4HC_Data_Structure; /************************************** Macros **************************************/ +#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(!!(c)) }; } /* Visual : use only *after* variable declarations */ #define LZ4_WILDCOPY(s,d,e) do { LZ4_COPYPACKET(s,d) } while (d> ((MINMATCH*8)-HASH_LOG)) @@ -342,87 +340,41 @@ FORCE_INLINE int LZ4_NbCommonBytes (register U32 val) #endif -int LZ4_sizeofStreamStateHC() -{ - return sizeof(LZ4HC_Data_Structure); -} - -FORCE_INLINE void LZ4_initHC (LZ4HC_Data_Structure* hc4, const BYTE* base) +FORCE_INLINE void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* base) { MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); - hc4->nextToUpdate = base + 1; + hc4->nextToUpdate = 1; hc4->base = base; hc4->inputBuffer = base; hc4->end = base; } -int LZ4_resetStreamStateHC(void* state, const char* inputBuffer) -{ - if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */ - LZ4_initHC((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer); - return 0; -} - - -void* LZ4_createHC (const char* inputBuffer) -{ - void* hc4 = ALLOCATOR(sizeof(LZ4HC_Data_Structure)); - LZ4_initHC ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer); - return hc4; -} - - -int LZ4_freeHC (void* LZ4HC_Data) -{ - FREEMEM(LZ4HC_Data); - return (0); -} - /* Update chains up to ip (excluded) */ FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip) { - U16* chainTable = hc4->chainTable; - HTYPE* HashTable = hc4->hashTable; - INITBASE(base,hc4->base); + U16* chainTable = hc4->chainTable; + U32* HashTable = hc4->hashTable; + const BYTE* const base = hc4->base; + const U32 target = ip - base; + U32 idx = hc4->nextToUpdate; - while(hc4->nextToUpdate < ip) + while(idx < target) { - const BYTE* const p = hc4->nextToUpdate; - size_t delta = (p) - HASH_POINTER(p); + U32 h = HASH_VALUE(base+idx); + size_t delta = idx - HashTable[h]; if (delta>MAX_DISTANCE) delta = MAX_DISTANCE; - DELTANEXT(p) = (U16)delta; - HashTable[HASH_VALUE(p)] = (HTYPE)((p) - base); - hc4->nextToUpdate++; + chainTable[idx & 0xFFFF] = (U16)delta; + HashTable[h] = idx; + idx++; } -} - - -char* LZ4_slideInputBufferHC(void* LZ4HC_Data) -{ - LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data; - size_t distance = (hc4->end - 64 KB) - hc4->inputBuffer; - - if (hc4->end <= hc4->inputBuffer + 64 KB) return (char*)(hc4->end); /* no update : less than 64KB within buffer */ - distance = (distance >> 16) << 16; /* Must be a multiple of 64 KB */ - LZ4HC_Insert(hc4, hc4->end - MINMATCH); - memcpy((void*)(hc4->end - 64 KB - distance), (const void*)(hc4->end - 64 KB), 64 KB); - hc4->nextToUpdate -= distance; - hc4->base -= distance; - if ((U32)(hc4->inputBuffer - hc4->base) > 1 GB + 64 KB) /* Avoid overflow */ - { - int i; - hc4->base += 1 GB; - for (i=0; ihashTable[i] -= 1 GB; - } - hc4->end -= distance; - return (char*)(hc4->end); + hc4->nextToUpdate = target; } -FORCE_INLINE size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* const matchlimit) +static size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* const matchlimit) { const BYTE* p1t = p1; @@ -443,124 +395,79 @@ FORCE_INLINE size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BY FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* const matchlimit, const BYTE** matchpos, const int maxNbAttempts) { U16* const chainTable = hc4->chainTable; - HTYPE* const HashTable = hc4->hashTable; - const BYTE* ref; - INITBASE(base,hc4->base); + U32* const HashTable = hc4->hashTable; + const BYTE* const base = hc4->base; + int matchIndex; + const int idxLow = (ip-base) > 64 KB ? (ip - base) - 64 KB : 0; + const BYTE* match; int nbAttempts=maxNbAttempts; - size_t repl=0, ml=0; - U16 delta=0; /* useless assignment, to remove an uninitialization warning */ + size_t ml=0; /* HC4 match finder */ LZ4HC_Insert(hc4, ip); - ref = HASH_POINTER(ip); - -#define REPEAT_OPTIMIZATION -#ifdef REPEAT_OPTIMIZATION - /* Detect repetitive sequences of length <= 4 */ - if ((U32)(ip-ref) <= 4) /* potential repetition */ - { - if (A32(ref) == A32(ip)) /* confirmed */ - { - delta = (U16)(ip-ref); - repl = ml = LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit) + MINMATCH; - *matchpos = ref; - } - ref = GETNEXT(ref); - } -#endif + matchIndex = HashTable[HASH_VALUE(ip)]; - while (((U32)(ip-ref) <= MAX_DISTANCE) && (nbAttempts)) + while ((matchIndex>idxLow) && (nbAttempts)) { nbAttempts--; - if (*(ref+ml) == *(ip+ml)) - if (A32(ref) == A32(ip)) - { - size_t mlt = LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit) + MINMATCH; - if (mlt > ml) { ml = mlt; *matchpos = ref; } - } - ref = GETNEXT(ref); - } - -#ifdef REPEAT_OPTIMIZATION - /* Complete table */ - if (repl) - { - const BYTE* ptr = ip; - const BYTE* end; - - end = ip + repl - (MINMATCH-1); - while(ptr < end-delta) + match = base + matchIndex; + if (*(match+ml) == *(ip+ml)) + if (A32(match) == A32(ip)) { - DELTANEXT(ptr) = delta; /* Pre-Load */ - ptr++; + size_t mlt = LZ4HC_CommonLength(ip+MINMATCH, match+MINMATCH, matchlimit) + MINMATCH; + if (mlt > ml) { ml = mlt; *matchpos = match; } } - do - { - DELTANEXT(ptr) = delta; - HashTable[HASH_VALUE(ptr)] = (HTYPE)((ptr) - base); /* Head of chain */ - ptr++; - } while(ptr < end); - hc4->nextToUpdate = end; + matchIndex -= chainTable[matchIndex & 0xFFFF]; } -#endif return (int)ml; } -FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* startLimit, const BYTE* matchlimit, int longest, const BYTE** matchpos, const BYTE** startpos, const int maxNbAttempts) +FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( + LZ4HC_Data_Structure* hc4, + const BYTE* ip, + const BYTE* startLimit, + const BYTE* matchlimit, + int longest, + const BYTE** matchpos, + const BYTE** startpos, + const int maxNbAttempts) { - U16* const chainTable = hc4->chainTable; - HTYPE* const HashTable = hc4->hashTable; - INITBASE(base,hc4->base); - const BYTE* ref; + U16* const chainTable = hc4->chainTable; + U32* const HashTable = hc4->hashTable; + const BYTE* const base = hc4->base; + const BYTE* match; + int matchIndex; + const int idxLow = (ip-base) > 64 KB ? (ip-base) - 64 KB : 0; int nbAttempts = maxNbAttempts; int delta = (int)(ip-startLimit); /* First Match */ LZ4HC_Insert(hc4, ip); - ref = HASH_POINTER(ip); + matchIndex = HashTable[HASH_VALUE(ip)]; - while (((U32)(ip-ref) <= MAX_DISTANCE) && (nbAttempts)) + while ((matchIndex>idxLow) && (nbAttempts)) { nbAttempts--; - if (*(startLimit + longest) == *(ref - delta + longest)) - if (A32(ref) == A32(ip)) + match = base + matchIndex; + if (*(startLimit + longest) == *(match - delta + longest)) + if (A32(match) == A32(ip)) { -#if 1 - const BYTE* reft = ref+MINMATCH; - const BYTE* ipt = ip+MINMATCH; - const BYTE* startt = ip; - - while (iptstartLimit) && (reft > hc4->inputBuffer) && (startt[-1] == reft[-1])) {startt--; reft--;} + while ((startt>startLimit) && (tmpMatch > hc4->inputBuffer) && (startt[-1] == tmpMatch[-1])) {startt--; tmpMatch--;} if ((ipt-startt) > longest) { longest = (int)(ipt-startt); - *matchpos = reft; + *matchpos = tmpMatch; *startpos = startt; } } - ref = GETNEXT(ref); + matchIndex -= chainTable[matchIndex & 0xFFFF]; } return longest; @@ -574,7 +481,7 @@ FORCE_INLINE int LZ4HC_encodeSequence ( BYTE** op, const BYTE** anchor, int matchLength, - const BYTE* ref, + const BYTE* match, limitedOutput_directive limitedOutputBuffer, BYTE* oend) { @@ -592,7 +499,7 @@ FORCE_INLINE int LZ4HC_encodeSequence ( LZ4_BLINDCOPY(*anchor, *op, length); /* Encode Offset */ - LZ4_WRITE_LITTLEENDIAN_16(*op,(U16)(*ip-ref)); + LZ4_WRITE_LITTLEENDIAN_16(*op,(U16)(*ip-match)); /* Encode MatchLength */ length = (int)(matchLength-MINMATCH); @@ -629,7 +536,7 @@ static int LZ4HC_compress_generic ( BYTE* op = (BYTE*) dest; BYTE* const oend = op + maxOutputSize; - const int maxNbAttempts = compressionLevel > MAX_COMPRESSION_LEVEL ? 1 << MAX_COMPRESSION_LEVEL : compressionLevel ? 1<<(compressionLevel-1) : 1< MAX_COMPRESSION_LEVEL) compressionLevel = MAX_COMPRESSION_LEVEL; + if (compressionLevel == 0) compressionLevel = LZ4HC_DEFAULT_COMPRESSIONLEVEL; + maxNbAttempts = 1 << compressionLevel; /* Ensure blocks follow each other */ if (ip != ctx->end) return 0; ctx->end += inputSize; @@ -794,7 +705,6 @@ _Search3: ml2 = ml3; goto _Search3; - } /* Encode Last Literals */ @@ -814,28 +724,18 @@ _Search3: int LZ4_compressHC2(const char* source, char* dest, int inputSize, int compressionLevel) { - void* ctx = LZ4_createHC(source); - int result; - if (ctx==NULL) return 0; - - result = LZ4HC_compress_generic (ctx, source, dest, inputSize, 0, compressionLevel, noLimit); - - LZ4_freeHC(ctx); - return result; + LZ4HC_Data_Structure ctx; + LZ4HC_init(&ctx, (const BYTE*)source); + return LZ4HC_compress_generic (&ctx, source, dest, inputSize, 0, compressionLevel, noLimit); } int LZ4_compressHC(const char* source, char* dest, int inputSize) { return LZ4_compressHC2(source, dest, inputSize, 0); } int LZ4_compressHC2_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel) { - void* ctx = LZ4_createHC(source); - int result; - if (ctx==NULL) return 0; - - result = LZ4HC_compress_generic (ctx, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput); - - LZ4_freeHC(ctx); - return result; + LZ4HC_Data_Structure ctx; + LZ4HC_init(&ctx, (const BYTE*)source); + return LZ4HC_compress_generic (&ctx, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput); } int LZ4_compressHC_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) @@ -853,7 +753,7 @@ int LZ4_sizeofStateHC() { return sizeof(LZ4HC_Data_Structure); } int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel) { if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ - LZ4_initHC ((LZ4HC_Data_Structure*)state, (const BYTE*)source); + LZ4HC_init ((LZ4HC_Data_Structure*)state, (const BYTE*)source); return LZ4HC_compress_generic (state, source, dest, inputSize, 0, compressionLevel, noLimit); } @@ -864,7 +764,7 @@ int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int int LZ4_compressHC2_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel) { if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ - LZ4_initHC ((LZ4HC_Data_Structure*)state, (const BYTE*)source); + LZ4HC_init ((LZ4HC_Data_Structure*)state, (const BYTE*)source); return LZ4HC_compress_generic (state, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput); } @@ -872,26 +772,124 @@ int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, c { return LZ4_compressHC2_limitedOutput_withStateHC (state, source, dest, inputSize, maxOutputSize, 0); } -/**************************** - Stream functions -****************************/ +/************************************** + Experimental Streaming Functions +**************************************/ +/* allocation */ +LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); } +int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { free(LZ4_streamHCPtr); return 0; }; -int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize) + +/* initialization */ +void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) { - return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, 0, noLimit); + LZ4_STATIC_ASSERT(sizeof(LZ4HC_Data_Structure) <= LZ4_STREAMHCSIZE); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ + ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base = NULL; + ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel = (unsigned)compressionLevel; } -int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel) +int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) { - return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, compressionLevel, noLimit); + LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) dictionary); + LZ4HC_Insert ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*)dictionary +(dictSize-3)); + ((LZ4HC_Data_Structure*) LZ4_streamHCPtr)->end = (const BYTE*)dictionary + dictSize; + return 1; +} + + +/* compression */ + +int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize) +{ + if (((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base == NULL) LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) source); + return LZ4HC_compress_generic (LZ4_streamHCPtr, source, dest, inputSize, 0, ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel, noLimit); +} + +int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize) +{ + if (((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base == NULL) LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) source); + return LZ4HC_compress_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel, limitedOutput); +} + + +/* dictionary saving */ + +int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize) +{ + LZ4HC_Data_Structure* sp = (LZ4HC_Data_Structure*)LZ4_streamHCPtr; + if (dictSize > 64 KB) dictSize = 64 KB; + if (dictSize < 0) dictSize = 0; + if (dictSize > (sp->end - sp->base)) dictSize = sp->end - sp->base; + memcpy(safeBuffer, sp->end - dictSize, dictSize); + LZ4_loadDictHC(LZ4_streamHCPtr, safeBuffer, dictSize); + return dictSize; } + + +/*********************************** + Deprecated Streaming functions +***********************************/ +int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; } + +int LZ4_resetStreamStateHC(void* state, const char* inputBuffer) +{ + if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */ + LZ4HC_init((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer); + return 0; +} + +void* LZ4_createHC (const char* inputBuffer) +{ + void* hc4 = ALLOCATOR(sizeof(LZ4HC_Data_Structure)); + LZ4HC_init ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer); + return hc4; +} + +int LZ4_freeHC (void* LZ4HC_Data) +{ + FREEMEM(LZ4HC_Data); + return (0); +} + +/* +int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize) +{ + return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, 0, noLimit); +} int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, 0, limitedOutput); } +*/ + +int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel) +{ + return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, compressionLevel, noLimit); +} int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel) { return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput); } + +char* LZ4_slideInputBufferHC(void* LZ4HC_Data) +{ + LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data; + size_t distance = (hc4->end - 64 KB) - hc4->inputBuffer; + + if (hc4->end <= hc4->inputBuffer + 64 KB) return (char*)(hc4->end); /* no update : less than 64KB within buffer */ + + distance = (distance >> 16) << 16; /* Must be a multiple of 64 KB */ + LZ4HC_Insert(hc4, hc4->end - MINMATCH); + memcpy((void*)(hc4->end - 64 KB - distance), (const void*)(hc4->end - 64 KB), 64 KB); + hc4->base -= distance; + if ((U32)(hc4->inputBuffer - hc4->base) > 1 GB + 64 KB) /* Avoid overflow */ + { + int i; + hc4->base += 1 GB; + for (i=0; ihashTable[i] -= 1 GB; + } + hc4->end -= distance; + return (char*)(hc4->end); +} diff --git a/lz4hc.h b/lz4hc.h index deb2394..29a05f5 100644 --- a/lz4hc.h +++ b/lz4hc.h @@ -74,7 +74,7 @@ int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize */ /* Note : -Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license) + Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license) */ @@ -101,13 +101,63 @@ They just use the externally allocated memory area instead of allocating their o */ + + +/************************************** + Experimental Streaming Functions +**************************************/ +#define LZ4_STREAMHCSIZE_U32 65546 +#define LZ4_STREAMHCSIZE (LZ4_STREAMHCSIZE_U32 * sizeof(unsigned int)) +typedef struct { unsigned int table[LZ4_STREAMHCSIZE_U32]; } LZ4_streamHC_t; + +/* +This structure allows static allocation of LZ4 HC streaming state. +State must then be initialized using LZ4_resetStreamHC() before first use. + +If you prefer dynamic allocation, please refer to functions below. +*/ + +LZ4_streamHC_t* LZ4_createStreamHC(void); +int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr); + +/* +These functions create and release memory for LZ4 HC streaming state. +Newly created states are already initialized. +Existing state space can be re-used anytime using LZ4_resetStreamHC(). +*/ + +void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); +int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize); + +int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize); +int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); + +int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int maxDictSize); + +/* +These functions compress data in successive blocks of any size, using previous blocks as dictionary. + +Before starting compression, state must be properly initialized, using LZ4_resetStreamHC(). +A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional). + +Then, use LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue() to compress each successive block. +They work like usual LZ4_compressHC() or LZ4_compressHC_limitedOutput(), but use previous memory blocks to improve compression. +Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression. + +If, for any reason, previous data block can't be preserved in memory for next block compression, +you can still preserve it by moving it to a safer place, +using LZ4_saveDictHC(). +*/ + + + /************************************** - Streaming Functions + Deprecated Streaming Functions **************************************/ /* Note : these streaming functions still follows the older model */ void* LZ4_createHC (const char* inputBuffer); -int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize); -int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize); +//int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize); +//int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize); char* LZ4_slideInputBufferHC (void* LZ4HC_Data); int LZ4_freeHC (void* LZ4HC_Data); diff --git a/programs/fullbench.c b/programs/fullbench.c index f34c68c..4caea16 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -313,12 +313,12 @@ static int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inS static int local_LZ4_compressHC_continue(const char* in, char* out, int inSize) { - return LZ4_compressHC_continue(ctx, in, out, inSize); + return LZ4_compressHC_continue((LZ4_streamHC_t*)ctx, in, out, inSize); } static int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize) { - return LZ4_compressHC_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize)); + return LZ4_compressHC_limitedOutput_continue((LZ4_streamHC_t*)ctx, in, out, inSize, LZ4_compressBound(inSize)); } static int local_LZ4F_compressFrame(const char* in, char* out, int inSize) diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 283352c..d736b63 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -621,14 +621,102 @@ _output_error: } } +static const unsigned testInputSize = 128 KB; +static const unsigned testCompressedSize = 64 KB; static void FUZ_unitTests(void) { + const unsigned testNb = 0; + const unsigned seed = 0; + const unsigned cycleNb= 0; + char testInput[testInputSize]; + char testCompressed[testCompressedSize]; + char testVerify[testCompressedSize]; + U32 randState = 0; + + // Init + FUZ_fillCompressibleNoiseBuffer(testInput, testInputSize, 0.50, &randState); + + // 32-bits address space overflow test FUZ_AddressOverflow(); + + // LZ4 HC streaming tests + { + LZ4_streamHC_t* sp; + LZ4_streamHC_t sHC; + //XXH64_state_t xxh; + U64 crcOrig; + U64 crcNew; + int result; + + // Allocation test + sp = LZ4_createStreamHC(); + FUZ_CHECKTEST(sp==NULL, "LZ4_createStreamHC() allocation failed"); + LZ4_freeStreamHC(sp); + + // simple compression test + crcOrig = XXH64(testInput, testCompressedSize, 0); + LZ4_resetStreamHC(&sHC, 0); + result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1); + FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); + + result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize); + FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); + crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); + + // simple dictionary compression test + crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); + LZ4_resetStreamHC(&sHC, 0); + LZ4_loadDictHC(&sHC, testInput, 64 KB); + result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); + FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); + + result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 64 KB); + FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() dictionary decompression failed"); + crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); + + // dictionary multi compression test + { + int result1, result2; + int segSize = testCompressedSize / 2; + crcOrig = XXH64(testInput + segSize, 64 KB, 0); + LZ4_resetStreamHC(&sHC, 0); + LZ4_loadDictHC(&sHC, testInput, segSize); + result1 = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1); + FUZ_CHECKTEST(result1==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); + result2 = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + 2*segSize, testCompressed+result1, segSize, segSize -1); + FUZ_CHECKTEST(result2==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); + + result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result1, segSize, testInput, segSize); + FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 1 failed"); + result = LZ4_decompress_safe_usingDict(testCompressed+result1, testVerify+segSize, result2, segSize, testInput, 2*segSize); + FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 2 failed"); + crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); + } + + // remote dictionary compression test + crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); + LZ4_resetStreamHC(&sHC, 0); + LZ4_loadDictHC(&sHC, testInput, 32 KB); + result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); + //FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result); + + result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 32 KB); + //FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() dictionary decompression failed"); + crcNew = XXH64(testVerify, testCompressedSize, 0); + //FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); + } + + return; +_output_error: + exit(1); } -int FUZ_usage(void) +static int FUZ_usage(void) { DISPLAY( "Usage :\n"); DISPLAY( " %s [args]\n", programName); -- cgit v0.12 From 68ea218a725351dde614950d1eaf0f8bc254a789 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 18 Oct 2014 13:27:32 +0100 Subject: fuzzer : fixed minor command lines issues --- programs/fuzzer.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/programs/fuzzer.c b/programs/fuzzer.c index d736b63..b0e042a 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -90,16 +90,17 @@ Macros *****************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static const U32 refreshRate = 250; +#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static int g_displayLevel = 2; +static const U32 g_refreshRate = 250; static U32 g_time = 0; /***************************************** - Local Parameters + Unit Variables *****************************************/ -static char* programName; -static int displayLevel = 2; +static char* programName = NULL; +static int g_pause = 0; /********************************************************* @@ -267,11 +268,11 @@ _overflowError: static void FUZ_displayUpdate(int testNb) { - if ((FUZ_GetMilliSpan(g_time) > refreshRate) || (displayLevel>=3)) + if ((FUZ_GetMilliSpan(g_time) > g_refreshRate) || (g_displayLevel>=3)) { g_time = FUZ_GetMilliStart(); DISPLAY("\r%5u ", testNb); - if (displayLevel>=3) fflush(stdout); + if (g_displayLevel>=3) fflush(stdout); } } @@ -292,7 +293,7 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili int ret, cycleNb; # define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %i : ", testNb); printf(__VA_ARGS__); \ printf(" (seed %u, cycle %i) \n", seed, cycleNb); goto _output_error; } -# define FUZ_DISPLAYTEST { testNb++; displayLevel<3 ? 0 : printf("%2i\b\b", testNb); if (displayLevel==4) fflush(stdout); } +# define FUZ_DISPLAYTEST { testNb++; g_displayLevel<3 ? 0 : printf("%2i\b\b", testNb); if (g_displayLevel==4) fflush(stdout); } void* stateLZ4 = malloc(LZ4_sizeofState()); void* stateLZ4HC = malloc(LZ4_sizeofStateHC()); void* LZ4continue; @@ -741,7 +742,6 @@ int main(int argc, char** argv) int nbTests = NB_ATTEMPTS; int testNb = 0; int proba = FUZ_COMPRESSIBILITY_DEFAULT; - int pause=0; // Check command line programName = argv[0]; @@ -754,11 +754,11 @@ int main(int argc, char** argv) // Decode command (note : aggregated commands are allowed) if (argument[0]=='-') { - if (!strcmp(argument, "--no-prompt")) { pause=0; seedset=1; displayLevel=1; continue; } + if (!strcmp(argument, "--no-prompt")) { g_pause=0; seedset=1; g_displayLevel=1; continue; } + argument++; - while (argument[1]!=0) + while (*argument!=0) { - argument++; switch(*argument) { case 'h': /* display help */ @@ -766,12 +766,12 @@ int main(int argc, char** argv) case 'v': /* verbose mode */ argument++; - displayLevel=4; + g_displayLevel=4; break; case 'p': /* pause at the end */ argument++; - pause=1; + g_pause=1; break; case 'i': @@ -832,13 +832,13 @@ int main(int argc, char** argv) printf("Seed = %u\n", seed); if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba); - FUZ_unitTests(); + if (seedset==0) FUZ_unitTests(); if (nbTests<=0) nbTests=1; { int result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100); - if (pause) + if (g_pause) { DISPLAY("press enter ... \n"); getchar(); -- cgit v0.12 From 6b89dca1a94ed47a9bd25b1da65f9a7c787da69d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 19 Oct 2014 13:47:22 +0100 Subject: fuzzer : faster jump to testNb --- programs/fuzzer.c | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/programs/fuzzer.c b/programs/fuzzer.c index b0e042a..eb5a3ff 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -100,7 +100,6 @@ static U32 g_time = 0; Unit Variables *****************************************/ static char* programName = NULL; -static int g_pause = 0; /********************************************************* @@ -126,8 +125,12 @@ static U32 FUZ_GetMilliSpan(U32 nTimeStart) } -# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) -unsigned int FUZ_rand(unsigned int* src) +static U32 FUZ_rotl32(U32 u32, U32 nbBits) +{ + return ((u32 << nbBits) | (u32 >> (32 - nbBits))); +} + +static U32 FUZ_rand(U32* src) { U32 rand32 = *src; rand32 *= PRIME1; @@ -140,7 +143,7 @@ unsigned int FUZ_rand(unsigned int* src) #define FUZ_RAND15BITS ((FUZ_rand(seed) >> 3) & 32767) #define FUZ_RANDLENGTH ( ((FUZ_rand(seed) >> 7) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15) -void FUZ_fillCompressibleNoiseBuffer(void* buffer, int bufferSize, double proba, U32* seed) +static void FUZ_fillCompressibleNoiseBuffer(void* buffer, int bufferSize, double proba, U32* seed) { BYTE* BBuffer = (BYTE*)buffer; int pos = 0; @@ -318,18 +321,23 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili char* dict; char* block; + FUZ_displayUpdate(cycleNb); + blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE; blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; if (dictSize > blockStart) dictSize = blockStart; block = ((char*)CNBuffer) + blockStart; dict = block - dictSize; - LZ4_loadDict(&LZ4dict, dict, dictSize); - LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); - LZ4_loadDict(&LZ4dict, dict, dictSize); - LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); - LZ4_loadDict(&LZ4dict, dict, dictSize); - LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); + if (0) // some problems related to dictionary re-use; in this case, enable this loop + { + LZ4_loadDict(&LZ4dict, dict, dictSize); + LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); + LZ4_loadDict(&LZ4dict, dict, dictSize); + LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); + LZ4_loadDict(&LZ4dict, dict, dictSize); + LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); + } } // Test loop @@ -742,6 +750,7 @@ int main(int argc, char** argv) int nbTests = NB_ATTEMPTS; int testNb = 0; int proba = FUZ_COMPRESSIBILITY_DEFAULT; + int pause = 0; // Check command line programName = argv[0]; @@ -754,7 +763,7 @@ int main(int argc, char** argv) // Decode command (note : aggregated commands are allowed) if (argument[0]=='-') { - if (!strcmp(argument, "--no-prompt")) { g_pause=0; seedset=1; g_displayLevel=1; continue; } + if (!strcmp(argument, "--no-prompt")) { pause=0; seedset=1; g_displayLevel=1; continue; } argument++; while (*argument!=0) @@ -771,7 +780,7 @@ int main(int argc, char** argv) case 'p': /* pause at the end */ argument++; - g_pause=1; + pause=1; break; case 'i': @@ -838,7 +847,7 @@ int main(int argc, char** argv) { int result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100); - if (g_pause) + if (pause) { DISPLAY("press enter ... \n"); getchar(); -- cgit v0.12 From f91d5aae792f4505199227d6b0d487ea83f667a7 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 19 Oct 2014 15:25:35 +0100 Subject: fuzzer : fixed test 21 --- programs/fuzzer.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/programs/fuzzer.c b/programs/fuzzer.c index eb5a3ff..4960e76 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -280,8 +280,6 @@ static void FUZ_displayUpdate(int testNb) } -#define FUZ_MAX(a,b) (a>b?a:b) - static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) { unsigned long long bytes = 0; @@ -499,10 +497,10 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili // Test HC compression with just one missing byte into output buffer => must fail FUZ_DISPLAYTEST; - compressedBuffer[compressedSize-1] = 0; + compressedBuffer[HCcompressedSize-1] = 0; ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize-1); FUZ_CHECKTEST(ret, "LZ4_compressHC_limitedOutput should have failed (output buffer too small by 1 byte)"); - FUZ_CHECKTEST(compressedBuffer[compressedSize-1], "LZ4_compressHC_limitedOutput overran output buffer") + FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-1], "LZ4_compressHC_limitedOutput overran output buffer") /* Dictionary tests */ -- cgit v0.12 From d6186730570999721eb8b6ae012e2b61265646e5 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 19 Oct 2014 16:06:33 +0100 Subject: fuzzer : easier random replay --- programs/fuzzer.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 4960e76..cc73262 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -96,12 +96,6 @@ static const U32 g_refreshRate = 250; static U32 g_time = 0; -/***************************************** - Unit Variables -*****************************************/ -static char* programName = NULL; - - /********************************************************* Fuzzer functions *********************************************************/ @@ -290,7 +284,6 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili char* compressedBuffer; char* decodedBuffer; # define FUZ_max LZ4_COMPRESSBOUND(LEN) - unsigned int randState=seed; int ret, cycleNb; # define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %i : ", testNb); printf(__VA_ARGS__); \ printf(" (seed %u, cycle %i) \n", seed, cycleNb); goto _output_error; } @@ -300,6 +293,8 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili void* LZ4continue; LZ4_stream_t LZ4dict; U32 crcOrig, crcCheck; + U32 coreRandState = seed; + U32 randState = coreRandState ^ PRIME3; // init @@ -319,16 +314,18 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili char* dict; char* block; - FUZ_displayUpdate(cycleNb); + (void)FUZ_rand(&coreRandState); - blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE; - blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); - dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; - if (dictSize > blockStart) dictSize = blockStart; - block = ((char*)CNBuffer) + blockStart; - dict = block - dictSize; if (0) // some problems related to dictionary re-use; in this case, enable this loop { + FUZ_displayUpdate(cycleNb); + randState = coreRandState ^ PRIME3; + blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE; + blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); + dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; + if (dictSize > blockStart) dictSize = blockStart; + block = ((char*)CNBuffer) + blockStart; + dict = block - dictSize; LZ4_loadDict(&LZ4dict, dict, dictSize); LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); LZ4_loadDict(&LZ4dict, dict, dictSize); @@ -348,6 +345,8 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili int blockContinueCompressedSize; FUZ_displayUpdate(cycleNb); + (void)FUZ_rand(&coreRandState); + randState = coreRandState ^ PRIME3; // Select block to test blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE; @@ -717,13 +716,14 @@ static void FUZ_unitTests(void) //FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); } + printf("All unit tests completed succesfully \n"); return; _output_error: exit(1); } -static int FUZ_usage(void) +static int FUZ_usage(char* programName) { DISPLAY( "Usage :\n"); DISPLAY( " %s [args]\n", programName); @@ -749,9 +749,9 @@ int main(int argc, char** argv) int testNb = 0; int proba = FUZ_COMPRESSIBILITY_DEFAULT; int pause = 0; + char* programName = argv[0]; // Check command line - programName = argv[0]; for(argNb=1; argNb Date: Sun, 19 Oct 2014 17:41:42 +0100 Subject: minor refactoring of LZ4 HC, to prepare for external dictionaries --- lz4hc.c | 72 ++++++++++++++++++++++++++++++++++++++++++++--------------------- lz4hc.h | 7 +++---- 2 files changed, 52 insertions(+), 27 deletions(-) diff --git a/lz4hc.c b/lz4hc.c index 2cf494e..5e204c6 100644 --- a/lz4hc.c +++ b/lz4hc.c @@ -253,8 +253,11 @@ typedef struct const BYTE* inputBuffer; const BYTE* base; const BYTE* end; + const BYTE* dictBase; + U32 dictLimit; U32 nextToUpdate; U32 compressionLevel; + U32 lowLimit; } LZ4HC_Data_Structure; @@ -348,6 +351,9 @@ FORCE_INLINE void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* base) hc4->base = base; hc4->inputBuffer = base; hc4->end = base; + hc4->dictBase = base; + hc4->dictLimit = 0; + hc4->lowLimit = 0; } @@ -374,31 +380,36 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip) } -static size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* const matchlimit) +static size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* const p1Limit) { - const BYTE* p1t = p1; + const BYTE* const p1Start = p1; - while (p1tchainTable; U32* const HashTable = hc4->hashTable; const BYTE* const base = hc4->base; - int matchIndex; - const int idxLow = (ip-base) > 64 KB ? (ip - base) - 64 KB : 0; + const BYTE* const dictBase = hc4->dictBase; + const U32 dictLimit = hc4->dictLimit; + U32 matchIndex; + const U32 idxLow = (ip-base) > 64 KB ? (ip - base) - 64 KB : 0; const BYTE* match; int nbAttempts=maxNbAttempts; size_t ml=0; @@ -410,11 +421,11 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const while ((matchIndex>idxLow) && (nbAttempts)) { nbAttempts--; - match = base + matchIndex; - if (*(match+ml) == *(ip+ml)) - if (A32(match) == A32(ip)) + match = ((matchIndex ml) { ml = mlt; *matchpos = match; } } matchIndex -= chainTable[matchIndex & 0xFFFF]; @@ -474,6 +485,19 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( } +static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock) +{ + LZ4HC_Insert (ctxPtr, ctxPtr->end-3); // finish referencing dictionary content + // Use only one memory segment for dict, so any previous External Dict is lost at this stage + ctxPtr->lowLimit = ctxPtr->dictLimit; + ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); + ctxPtr->dictBase = ctxPtr->base; + ctxPtr->base = newBlock - ctxPtr->dictLimit; + ctxPtr->end = newBlock; + ctxPtr->nextToUpdate = ctxPtr->dictLimit; // reference table must skip to from beginning of block +} + + typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive; FORCE_INLINE int LZ4HC_encodeSequence ( @@ -488,10 +512,12 @@ FORCE_INLINE int LZ4HC_encodeSequence ( int length; BYTE* token; + //printf("literal : %u -- match : %u \n", (U32)(*ip - *anchor), (U32)matchLength); // debug + /* Encode Literal length */ length = (int)(*ip - *anchor); token = (*op)++; - if ((limitedOutputBuffer) && ((*op + length + (2 + 1 + LASTLITERALS) + (length>>8)) > oend)) return 1; /* Check output limit */ + if ((limitedOutputBuffer) && ((*op + (length>>8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */ if (length>=(int)RUN_MASK) { int len; *token=(RUN_MASK< 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; } else *token = (BYTE)(length<>8) > oend)) return 1; /* Check output limit */ + if ((limitedOutputBuffer) && (*op + (length>>8) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */ if (length>=(int)ML_MASK) { *token+=ML_MASK; length-=ML_MASK; for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; } if (length > 254) { length-=255; *(*op)++ = 255; } *(*op)++ = (BYTE)length; } else *token += (BYTE)(length); @@ -551,8 +577,8 @@ static int LZ4HC_compress_generic ( if (compressionLevel > MAX_COMPRESSION_LEVEL) compressionLevel = MAX_COMPRESSION_LEVEL; if (compressionLevel == 0) compressionLevel = LZ4HC_DEFAULT_COMPRESSIONLEVEL; maxNbAttempts = 1 << compressionLevel; - /* Ensure blocks follow each other */ - if (ip != ctx->end) return 0; + /* check if blocks follow each other */ + if (ip != ctx->end) LZ4HC_setExternalDict(ctx, ip); ctx->end += inputSize; ip++; @@ -747,7 +773,7 @@ int LZ4_compressHC_limitedOutput(const char* source, char* dest, int inputSize, /***************************** Using external allocation *****************************/ -int LZ4_sizeofStateHC() { return sizeof(LZ4HC_Data_Structure); } +int LZ4_sizeofStateHC(void) { return sizeof(LZ4HC_Data_Structure); } int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel) diff --git a/lz4hc.h b/lz4hc.h index 29a05f5..72979b5 100644 --- a/lz4hc.h +++ b/lz4hc.h @@ -95,18 +95,17 @@ int LZ4_sizeofStateHC(); Note that tables must be aligned for pointer (32 or 64 bits), otherwise compression will fail (return code 0). -The allocated memory can be provided to the compressions functions using 'void* state' parameter. +The allocated memory can be provided to the compression functions using 'void* state' parameter. LZ4_compress_withStateHC() and LZ4_compress_limitedOutput_withStateHC() are equivalent to previously described functions. -They just use the externally allocated memory area instead of allocating their own (on stack, or on heap). +They just use the externally allocated memory for state instead of allocating their own (on stack, or on heap). */ - /************************************** Experimental Streaming Functions **************************************/ -#define LZ4_STREAMHCSIZE_U32 65546 +#define LZ4_STREAMHCSIZE_U32 65548 #define LZ4_STREAMHCSIZE (LZ4_STREAMHCSIZE_U32 * sizeof(unsigned int)) typedef struct { unsigned int table[LZ4_STREAMHCSIZE_U32]; } LZ4_streamHC_t; -- cgit v0.12 From 3dab5f476a2e5a0cd4cd9a859e94a5110abda23d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 20 Oct 2014 01:08:21 +0100 Subject: LZ4 HC : External Dictionary compression : First implementation --- lz4hc.c | 135 +++++++++++++++++++++++++++++++++++------------------- lz4hc.h | 0 programs/fuzzer.c | 100 +++++++++++++++++++++++++++------------- 3 files changed, 156 insertions(+), 79 deletions(-) mode change 100644 => 100755 lz4hc.h diff --git a/lz4hc.c b/lz4hc.c index 5e204c6..72739a7 100644 --- a/lz4hc.c +++ b/lz4hc.c @@ -268,11 +268,10 @@ typedef struct #define LZ4_WILDCOPY(s,d,e) do { LZ4_COPYPACKET(s,d) } while (d> ((MINMATCH*8)-HASH_LOG)) -#define HASH_VALUE(p) HASH_FUNCTION(A32(p)) -#define HASH_POINTER(p) (HashTable[HASH_VALUE(p)] + base) #define DELTANEXT(p) chainTable[(size_t)(p) & MAXD_MASK] #define GETNEXT(p) ((p) - (size_t)DELTANEXT(p)) +static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(A32(ptr)); } /************************************** Private functions @@ -363,12 +362,12 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip) U16* chainTable = hc4->chainTable; U32* HashTable = hc4->hashTable; const BYTE* const base = hc4->base; - const U32 target = ip - base; + const U32 target = (U32)(ip - base); U32 idx = hc4->nextToUpdate; while(idx < target) { - U32 h = HASH_VALUE(base+idx); + U32 h = LZ4HC_hashPtr(base+idx); size_t delta = idx - HashTable[h]; if (delta>MAX_DISTANCE) delta = MAX_DISTANCE; chainTable[idx & 0xFFFF] = (U16)delta; @@ -398,10 +397,25 @@ static size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* co } -FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, - const BYTE* ip, const BYTE* iLimit, - const BYTE** matchpos, - const int maxNbAttempts) +static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock) +{ + if (ctxPtr->end >= ctxPtr->base + 4) + LZ4HC_Insert (ctxPtr, ctxPtr->end-3); // finish referencing dictionary content + // Note : need to handle risk of index overflow + // Use only one memory segment for dict, so any previous External Dict is lost at this stage + ctxPtr->lowLimit = ctxPtr->dictLimit; + ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); + ctxPtr->dictBase = ctxPtr->base; + ctxPtr->base = newBlock - ctxPtr->dictLimit; + ctxPtr->end = newBlock; + ctxPtr->nextToUpdate = ctxPtr->dictLimit; // reference table must skip to from beginning of block +} + + +FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, // Index table will be updated + const BYTE* ip, const BYTE* const iLimit, + const BYTE** matchpos, + const int maxNbAttempts) { U16* const chainTable = hc4->chainTable; U32* const HashTable = hc4->hashTable; @@ -409,24 +423,41 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const BYTE* const dictBase = hc4->dictBase; const U32 dictLimit = hc4->dictLimit; U32 matchIndex; - const U32 idxLow = (ip-base) > 64 KB ? (ip - base) - 64 KB : 0; + const U32 idxLow = (ip-base) > 64 KB ? (U32)(ip - base) - 64 KB : 0; const BYTE* match; int nbAttempts=maxNbAttempts; size_t ml=0; /* HC4 match finder */ LZ4HC_Insert(hc4, ip); - matchIndex = HashTable[HASH_VALUE(ip)]; + matchIndex = HashTable[LZ4HC_hashPtr(ip)]; while ((matchIndex>idxLow) && (nbAttempts)) { nbAttempts--; - match = ((matchIndex= dictLimit) { - size_t mlt = LZ4HC_CommonLength(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; - if (mlt > ml) { ml = mlt; *matchpos = match; } + match = base + matchIndex; + if (*(match+ml) == *(ip+ml) + && (A32(match) == A32(ip))) + { + size_t mlt = LZ4HC_CommonLength(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + if (mlt > ml) { ml = mlt; *matchpos = match; } + } + } + else + { + match = dictBase + matchIndex; + if (A32(match) == A32(ip)) + { + size_t mlt; + const BYTE* vLimit = ip + (dictLimit - matchIndex); + if (vLimit > iLimit) vLimit = iLimit; + mlt = LZ4HC_CommonLength(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; + if ((ip+mlt == vLimit) && (vLimit < iLimit)) + mlt += LZ4HC_CommonLength(ip+mlt, base+dictLimit, iLimit); + if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } // virtual matchpos + } } matchIndex -= chainTable[matchIndex & 0xFFFF]; } @@ -438,8 +469,8 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( LZ4HC_Data_Structure* hc4, const BYTE* ip, - const BYTE* startLimit, - const BYTE* matchlimit, + const BYTE* iLowLimit, + const BYTE* iHighLimit, int longest, const BYTE** matchpos, const BYTE** startpos, @@ -448,34 +479,57 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( U16* const chainTable = hc4->chainTable; U32* const HashTable = hc4->hashTable; const BYTE* const base = hc4->base; + const U32 dictLimit = hc4->dictLimit; + const U32 dictLowLimit = hc4->lowLimit; + const BYTE* const dictBase = hc4->dictBase; const BYTE* match; - int matchIndex; - const int idxLow = (ip-base) > 64 KB ? (ip-base) - 64 KB : 0; + U32 matchIndex; + const U32 idxLow = (ip-base) > 64 KB ? (U32)(ip-base) - 64 KB : 0; int nbAttempts = maxNbAttempts; - int delta = (int)(ip-startLimit); + int delta = (int)(ip-iLowLimit); /* First Match */ LZ4HC_Insert(hc4, ip); - matchIndex = HashTable[HASH_VALUE(ip)]; + matchIndex = HashTable[LZ4HC_hashPtr(ip)]; while ((matchIndex>idxLow) && (nbAttempts)) { nbAttempts--; - match = base + matchIndex; - if (*(startLimit + longest) == *(match - delta + longest)) - if (A32(match) == A32(ip)) + if (matchIndex >= dictLimit) { - const BYTE* startt = ip; - const BYTE* tmpMatch = match; - const BYTE* ipt = ip + MINMATCH + LZ4HC_CommonLength(ip+MINMATCH, match+MINMATCH, matchlimit); + match = base + matchIndex; + if (*(iLowLimit + longest) == *(match - delta + longest)) + if (A32(match) == A32(ip)) + { + const BYTE* startt = ip; + const BYTE* tmpMatch = match; + const BYTE* const matchEnd = ip + MINMATCH + LZ4HC_CommonLength(ip+MINMATCH, match+MINMATCH, iHighLimit); - while ((startt>startLimit) && (tmpMatch > hc4->inputBuffer) && (startt[-1] == tmpMatch[-1])) {startt--; tmpMatch--;} + while ((startt>iLowLimit) && (tmpMatch > iLowLimit) && (startt[-1] == tmpMatch[-1])) {startt--; tmpMatch--;} - if ((ipt-startt) > longest) + if ((matchEnd-startt) > longest) + { + longest = (int)(matchEnd-startt); + *matchpos = tmpMatch; + *startpos = startt; + } + } + } + else + { + match = dictBase + matchIndex; + if (A32(match) == A32(ip)) { - longest = (int)(ipt-startt); - *matchpos = tmpMatch; - *startpos = startt; + size_t mlt; + int back=0; + const BYTE* vLimit = ip + (dictLimit - matchIndex); + if (vLimit > iHighLimit) vLimit = iHighLimit; + mlt = LZ4HC_CommonLength(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; + if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) + mlt += LZ4HC_CommonLength(ip+mlt, base+dictLimit, iHighLimit); + while ((ip+back > iLowLimit) && (matchIndex+back > dictLowLimit) && (ip[back-1] == match[back-1])) back--; + mlt -= back; + if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; } } } matchIndex -= chainTable[matchIndex & 0xFFFF]; @@ -485,19 +539,6 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( } -static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock) -{ - LZ4HC_Insert (ctxPtr, ctxPtr->end-3); // finish referencing dictionary content - // Use only one memory segment for dict, so any previous External Dict is lost at this stage - ctxPtr->lowLimit = ctxPtr->dictLimit; - ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); - ctxPtr->dictBase = ctxPtr->base; - ctxPtr->base = newBlock - ctxPtr->dictLimit; - ctxPtr->end = newBlock; - ctxPtr->nextToUpdate = ctxPtr->dictLimit; // reference table must skip to from beginning of block -} - - typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive; FORCE_INLINE int LZ4HC_encodeSequence ( @@ -817,7 +858,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) { LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) dictionary); - LZ4HC_Insert ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*)dictionary +(dictSize-3)); + if (dictSize >= 4) LZ4HC_Insert ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*)dictionary +(dictSize-3)); ((LZ4HC_Data_Structure*) LZ4_streamHCPtr)->end = (const BYTE*)dictionary + dictSize; return 1; } @@ -845,7 +886,7 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS LZ4HC_Data_Structure* sp = (LZ4HC_Data_Structure*)LZ4_streamHCPtr; if (dictSize > 64 KB) dictSize = 64 KB; if (dictSize < 0) dictSize = 0; - if (dictSize > (sp->end - sp->base)) dictSize = sp->end - sp->base; + if (dictSize > (sp->end - sp->base)) dictSize = (int)(sp->end - sp->base); memcpy(safeBuffer, sp->end - dictSize, dictSize); LZ4_loadDictHC(LZ4_streamHCPtr, safeBuffer, dictSize); return dictSize; diff --git a/lz4hc.h b/lz4hc.h old mode 100644 new mode 100755 diff --git a/programs/fuzzer.c b/programs/fuzzer.c index cc73262..cf56251 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -263,7 +263,7 @@ _overflowError: } -static void FUZ_displayUpdate(int testNb) +static void FUZ_displayUpdate(unsigned testNb) { if ((FUZ_GetMilliSpan(g_time) > g_refreshRate) || (g_displayLevel>=3)) { @@ -274,7 +274,7 @@ static void FUZ_displayUpdate(int testNb) } -static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) +static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const double compressibility) { unsigned long long bytes = 0; unsigned long long cbytes = 0; @@ -283,15 +283,17 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili void* CNBuffer; char* compressedBuffer; char* decodedBuffer; -# define FUZ_max LZ4_COMPRESSBOUND(LEN) - int ret, cycleNb; -# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %i : ", testNb); printf(__VA_ARGS__); \ - printf(" (seed %u, cycle %i) \n", seed, cycleNb); goto _output_error; } -# define FUZ_DISPLAYTEST { testNb++; g_displayLevel<3 ? 0 : printf("%2i\b\b", testNb); if (g_displayLevel==4) fflush(stdout); } +# define FUZ_max LZ4_COMPRESSBOUND(LEN) + int ret; + unsigned cycleNb; +# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \ + printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; } +# define FUZ_DISPLAYTEST { testNb++; g_displayLevel<3 ? 0 : printf("%2u\b\b", testNb); if (g_displayLevel==4) fflush(stdout); } void* stateLZ4 = malloc(LZ4_sizeofState()); void* stateLZ4HC = malloc(LZ4_sizeofStateHC()); void* LZ4continue; LZ4_stream_t LZ4dict; + LZ4_streamHC_t LZ4dictHC; U32 crcOrig, crcCheck; U32 coreRandState = seed; U32 randState = coreRandState ^ PRIME3; @@ -303,21 +305,19 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili // Create compressible test buffer CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); - compressedBuffer = malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE)); - decodedBuffer = malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE); + compressedBuffer = (char*)malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE)); + decodedBuffer = (char*)malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE); // move to startCycle for (cycleNb = 0; cycleNb < startCycle; cycleNb++) { - // synd rand & dict - int dictSize, blockSize, blockStart; - char* dict; - char* block; - (void)FUZ_rand(&coreRandState); if (0) // some problems related to dictionary re-use; in this case, enable this loop { + int dictSize, blockSize, blockStart; + char* dict; + char* block; FUZ_displayUpdate(cycleNb); randState = coreRandState ^ PRIME3; blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE; @@ -338,7 +338,7 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili // Test loop for (cycleNb = startCycle; cycleNb < nbCycles; cycleNb++) { - int testNb = 0; + U32 testNb = 0; char* dict; char* block; int dictSize, blockSize, blockStart, compressedSize, HCcompressedSize; @@ -506,8 +506,8 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili // Compress using dictionary FUZ_DISPLAYTEST; LZ4continue = LZ4_create (dict); - LZ4_compress_continue (LZ4continue, dict, compressedBuffer, dictSize); // Just to fill hash tables - blockContinueCompressedSize = LZ4_compress_continue (LZ4continue, block, compressedBuffer, blockSize); + LZ4_compress_continue ((LZ4_stream_t*)LZ4continue, dict, compressedBuffer, dictSize); // Just to fill hash tables + blockContinueCompressedSize = LZ4_compress_continue ((LZ4_stream_t*)LZ4continue, block, compressedBuffer, blockSize); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed"); free (LZ4continue); @@ -534,7 +534,7 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili // Compress using External dictionary FUZ_DISPLAYTEST; - dict -= 9; // Separation, so it is an ExtDict + dict -= (FUZ_rand(&randState) & 0xF) + 1; // Separation, so it is an ExtDict if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; LZ4_loadDict(&LZ4dict, dict, dictSize); blockContinueCompressedSize = LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); @@ -563,7 +563,6 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili int i=0; while (block[i]==decodedBuffer[i]) i++; printf("Wrong Byte at position %i/%i\n", i, blockSize); - } FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); @@ -588,23 +587,60 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); FUZ_DISPLAYTEST; - if (blockSize > 10) { - decodedBuffer[blockSize-10] = 0; - ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-10, dict, dictSize); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-10 byte)"); - FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-10 byte) (blockSize=%i)", blockSize); + U32 missingBytes = (FUZ_rand(&randState) & 0xF) + 2; + if ((U32)blockSize > missingBytes) + { + decodedBuffer[blockSize-missingBytes] = 0; + ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-missingBytes, dict, dictSize); + FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%u byte)", missingBytes); + FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize); + } + } + + // Compress HC using External dictionary + FUZ_DISPLAYTEST; + dict -= (FUZ_rand(&randState) & 7); // even bigger separation + if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; + LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); + blockContinueCompressedSize = LZ4_compressHC_continue(&LZ4dictHC, block, compressedBuffer, blockSize); + FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compressHC_continue failed"); + + FUZ_DISPLAYTEST; + LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); + ret = LZ4_compressHC_limitedOutput_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); + FUZ_CHECKTEST(ret>0, "LZ4_compressHC_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer"); + + FUZ_DISPLAYTEST; + LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); + ret = LZ4_compressHC_limitedOutput_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize); + FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); + FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer"); + + FUZ_DISPLAYTEST; + decodedBuffer[blockSize] = 0; + ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); + FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); + FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size") + crcCheck = XXH32(decodedBuffer, blockSize, 0); + if (crcCheck!=crcOrig) + { + int i=0; + while (block[i]==decodedBuffer[i]) i++; + printf("Wrong Byte at position %i/%i\n", i, blockSize); } + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); - // Fill stats + // ***** End of tests *** // + // Fill stats bytes += blockSize; cbytes += compressedSize; hcbytes += HCcompressedSize; ccbytes += blockContinueCompressedSize; } - printf("\r%7i /%7i - ", cycleNb, nbCycles); + printf("\r%7u /%7u - ", cycleNb, nbCycles); printf("all tests completed successfully \n"); printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100); printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100); @@ -627,8 +663,8 @@ _output_error: } } -static const unsigned testInputSize = 128 KB; -static const unsigned testCompressedSize = 64 KB; +#define testInputSize (128 KB) +#define testCompressedSize (64 KB) static void FUZ_unitTests(void) { @@ -703,17 +739,17 @@ static void FUZ_unitTests(void) FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); } - // remote dictionary compression test + // remote dictionary HC compression test crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); LZ4_resetStreamHC(&sHC, 0); LZ4_loadDictHC(&sHC, testInput, 32 KB); result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); - //FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result); + FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result); result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 32 KB); - //FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() dictionary decompression failed"); + FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe_usingDict() decompression failed following remote dictionary HC compression test"); crcNew = XXH64(testVerify, testCompressedSize, 0); - //FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() decompression corruption"); } printf("All unit tests completed succesfully \n"); -- cgit v0.12 From e450018588560537c2c4b4b2dd3515a9ef3a83f7 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 21 Oct 2014 00:12:55 +0100 Subject: LZ4 HC : fixed small dictionary streaming compression --- lz4.c | 6 +++--- lz4hc.c | 26 ++++++++++++------------ programs/fuzzer.c | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 70 insertions(+), 21 deletions(-) mode change 100644 => 100755 lz4.c mode change 100644 => 100755 lz4hc.c mode change 100644 => 100755 programs/fuzzer.c diff --git a/lz4.c b/lz4.c old mode 100644 new mode 100755 index 39f176f..8adf96a --- a/lz4.c +++ b/lz4.c @@ -876,8 +876,8 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) * in order to remove useless branches during compilation optimization. */ FORCE_INLINE int LZ4_decompress_generic( - const char* source, - char* dest, + const char* const source, + char* const dest, int inputSize, int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */ @@ -885,7 +885,7 @@ FORCE_INLINE int LZ4_decompress_generic( int partialDecoding, /* full, partial */ int targetOutputSize, /* only used if partialDecoding==partial */ int dict, /* noDict, withPrefix64k, usingExtDict */ - const char* dictStart, /* only if dict==usingExtDict */ + const char* const dictStart, /* only if dict==usingExtDict */ int dictSize /* note : = 0 if noDict */ ) { diff --git a/lz4hc.c b/lz4hc.c old mode 100644 new mode 100755 index 72739a7..38e0ce4 --- a/lz4hc.c +++ b/lz4hc.c @@ -346,13 +346,13 @@ FORCE_INLINE void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* base) { MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); - hc4->nextToUpdate = 1; - hc4->base = base; + hc4->nextToUpdate = 64 KB; + hc4->base = base - 64 KB; hc4->inputBuffer = base; hc4->end = base; - hc4->dictBase = base; - hc4->dictLimit = 0; - hc4->lowLimit = 0; + hc4->dictBase = base - 64 KB; + hc4->dictLimit = 64 KB; + hc4->lowLimit = 64 KB; } @@ -422,17 +422,17 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, // I const BYTE* const base = hc4->base; const BYTE* const dictBase = hc4->dictBase; const U32 dictLimit = hc4->dictLimit; + const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); U32 matchIndex; - const U32 idxLow = (ip-base) > 64 KB ? (U32)(ip - base) - 64 KB : 0; const BYTE* match; int nbAttempts=maxNbAttempts; size_t ml=0; - /* HC4 match finder */ + /* HC4 match finder */ LZ4HC_Insert(hc4, ip); matchIndex = HashTable[LZ4HC_hashPtr(ip)]; - while ((matchIndex>idxLow) && (nbAttempts)) + while ((matchIndex>=lowLimit) && (nbAttempts)) { nbAttempts--; if (matchIndex >= dictLimit) @@ -480,19 +480,19 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( U32* const HashTable = hc4->hashTable; const BYTE* const base = hc4->base; const U32 dictLimit = hc4->dictLimit; - const U32 dictLowLimit = hc4->lowLimit; + const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); const BYTE* const dictBase = hc4->dictBase; const BYTE* match; U32 matchIndex; - const U32 idxLow = (ip-base) > 64 KB ? (U32)(ip-base) - 64 KB : 0; int nbAttempts = maxNbAttempts; int delta = (int)(ip-iLowLimit); - /* First Match */ + + /* First Match */ LZ4HC_Insert(hc4, ip); matchIndex = HashTable[LZ4HC_hashPtr(ip)]; - while ((matchIndex>idxLow) && (nbAttempts)) + while ((matchIndex>=lowLimit) && (nbAttempts)) { nbAttempts--; if (matchIndex >= dictLimit) @@ -527,7 +527,7 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( mlt = LZ4HC_CommonLength(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) mlt += LZ4HC_CommonLength(ip+mlt, base+dictLimit, iHighLimit); - while ((ip+back > iLowLimit) && (matchIndex+back > dictLowLimit) && (ip[back-1] == match[back-1])) back--; + while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == match[back-1])) back--; mlt -= back; if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; } } diff --git a/programs/fuzzer.c b/programs/fuzzer.c old mode 100644 new mode 100755 index cf56251..225361b --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -673,7 +673,7 @@ static void FUZ_unitTests(void) const unsigned cycleNb= 0; char testInput[testInputSize]; char testCompressed[testCompressedSize]; - char testVerify[testCompressedSize]; + char testVerify[testInputSize]; U32 randState = 0; // Init @@ -719,7 +719,7 @@ static void FUZ_unitTests(void) crcNew = XXH64(testVerify, testCompressedSize, 0); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); - // dictionary multi compression test + // multiple HC compression test with dictionary { int result1, result2; int segSize = testCompressedSize / 2; @@ -727,9 +727,9 @@ static void FUZ_unitTests(void) LZ4_resetStreamHC(&sHC, 0); LZ4_loadDictHC(&sHC, testInput, segSize); result1 = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1); - FUZ_CHECKTEST(result1==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); - result2 = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + 2*segSize, testCompressed+result1, segSize, segSize -1); - FUZ_CHECKTEST(result2==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); + FUZ_CHECKTEST(result1==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result1); + result2 = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + 2*segSize, testCompressed+result1, segSize, segSize-1); + FUZ_CHECKTEST(result2==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result2); result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result1, segSize, testInput, segSize); FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 1 failed"); @@ -750,6 +750,55 @@ static void FUZ_unitTests(void) FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe_usingDict() decompression failed following remote dictionary HC compression test"); crcNew = XXH64(testVerify, testCompressedSize, 0); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() decompression corruption"); + + // multiple HC compression with ext. dictionary + { + XXH64_state_t crcOrigState; + XXH64_state_t crcNewState; + const char* dict = testInput + 3; + int dictSize = (FUZ_rand(&randState) & 8191); + char* dst = testVerify; + + size_t segStart = dictSize + 7; + int segSize = (FUZ_rand(&randState) & 8191); + int segNb = 1; + + LZ4_resetStreamHC(&sHC, 0); + LZ4_loadDictHC(&sHC, dict, dictSize); + + XXH64_reset(&crcOrigState, 0); + XXH64_reset(&crcNewState, 0); + + while (segStart + segSize < testInputSize) + { + XXH64_update(&crcOrigState, testInput + segStart, segSize); + crcOrig = XXH64_digest(&crcOrigState); + result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + segStart, testCompressed, segSize, LZ4_compressBound(segSize)); + FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); + + result = LZ4_decompress_safe_usingDict(testCompressed, dst, result, segSize, dict, dictSize); + FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %i failed", segNb); + XXH64_update(&crcNewState, dst, segSize); + crcNew = XXH64_digest(&crcNewState); + if (crcOrig!=crcNew) + { + size_t c=0; + while (dst[c] == testInput[segStart+c]) c++; + DISPLAY("Bad decompression at %u / %u \n", (U32)c, (U32)segSize); + } + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() part %i corruption", segNb); + + dict = dst; + //dict = testInput + segStart; + dictSize = segSize; + + dst += segSize + 1; + segNb ++; + + segStart += segSize + (FUZ_rand(&randState) & 0xF) + 1; + segSize = (FUZ_rand(&randState) & 8191); + } + } } printf("All unit tests completed succesfully \n"); -- cgit v0.12 From e468a4eb96ec8e2ea0fc61b5aa20f56a3b214c0e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 22 Oct 2014 00:36:32 +0100 Subject: fixed LZ4F_compressFrameBound (dynamic block resize) --- lz4frame.c | 14 ++++++++++++++ programs/frametest.c | 15 ++++++++------- 2 files changed, 22 insertions(+), 7 deletions(-) mode change 100644 => 100755 lz4frame.c mode change 100644 => 100755 programs/frametest.c diff --git a/lz4frame.c b/lz4frame.c old mode 100644 new mode 100755 index 93069e0..f7bf242 --- a/lz4frame.c +++ b/lz4frame.c @@ -234,6 +234,20 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere size_t streamSize; if (preferencesPtr!=NULL) prefs = *preferencesPtr; + { + blockSizeID_t proposedBSID = max64KB; + size_t maxBlockSize = 64 KB; + while (prefs.frameInfo.blockSizeID > proposedBSID) + { + if (srcSize <= maxBlockSize) + { + prefs.frameInfo.blockSizeID = proposedBSID; + break; + } + proposedBSID++; + maxBlockSize <<= 2; + } + } prefs.autoFlush = 1; headerSize = 7; /* basic header size (no option) including magic number */ diff --git a/programs/frametest.c b/programs/frametest.c old mode 100644 new mode 100755 index 343a8ab..abf0cde --- a/programs/frametest.c +++ b/programs/frametest.c @@ -78,7 +78,7 @@ typedef unsigned long long U64; #define MB *(1U<<20) #define GB *(1U<<30) -static const U32 nbTestsDefault = 128 KB; +static const U32 nbTestsDefault = 256 KB; #define COMPRESSIBLE_NOISE_LENGTH (2 MB) #define FUZ_COMPRESSIBILITY_DEFAULT 50 static const U32 prime1 = 2654435761U; @@ -597,7 +597,12 @@ int main(int argc, char** argv) argument++; displayLevel--; break; - case 'i': + case 'p': /* pause at the end */ + argument++; + pause = 1; + break; + + case 'i': argument++; nbTests=0; while ((*argument>='0') && (*argument<='9')) @@ -628,7 +633,7 @@ int main(int argc, char** argv) argument++; } break; - case 'p': /* compressibility % */ + case 'P': /* compressibility % */ argument++; proba=0; while ((*argument>='0') && (*argument<='9')) @@ -640,10 +645,6 @@ int main(int argc, char** argv) if (proba<0) proba=0; if (proba>100) proba=100; break; - case 'P': /* pause at the end */ - argument++; - pause = 1; - break; default: ; return FUZ_usage(); -- cgit v0.12 From e2c84118f52cefe48fd2f751e66ad3ecd904f7b9 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 22 Oct 2014 08:07:56 +0100 Subject: Added : HC streaming ring buffer example --- examples/HCStreaming_ringBuffer.c | 208 ++++++++++++++++++++++++++++++++++++++ examples/Makefile | 8 +- lz4hc.c | 70 +++++++++---- 3 files changed, 263 insertions(+), 23 deletions(-) create mode 100755 examples/HCStreaming_ringBuffer.c diff --git a/examples/HCStreaming_ringBuffer.c b/examples/HCStreaming_ringBuffer.c new file mode 100755 index 0000000..3532fa3 --- /dev/null +++ b/examples/HCStreaming_ringBuffer.c @@ -0,0 +1,208 @@ +// LZ4 HC streaming API example : ring buffer +// Based on previous work from Takayuki Matsuoka + + +/************************************** + Compiler Options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define _CRT_SECURE_NO_WARNINGS // for MSVC +# define snprintf sprintf_s +#endif + +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +#ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ +#endif + + + +/************************************** + Includes +**************************************/ +#include "lz4hc.h" +#include "lz4.h" + +#include +#include +#include +#include + +enum { + MESSAGE_MAX_BYTES = 1024, + RING_BUFFER_BYTES = 1024 * 256 + MESSAGE_MAX_BYTES, + DICT_BYTES = 65536, +}; + + +size_t write_int32(FILE* fp, int32_t i) { + return fwrite(&i, sizeof(i), 1, fp); +} + +size_t write_bin(FILE* fp, const void* array, int arrayBytes) { + return fwrite(array, 1, arrayBytes, fp); +} + +size_t read_int32(FILE* fp, int32_t* i) { + return fread(i, sizeof(*i), 1, fp); +} + +size_t read_bin(FILE* fp, void* array, int arrayBytes) { + return fread(array, 1, arrayBytes, fp); +} + + +void test_compress(FILE* outFp, FILE* inpFp) +{ + LZ4_streamHC_t lz4Stream_body = { 0 }; + LZ4_streamHC_t* lz4Stream = &lz4Stream_body; + + static char inpBuf[RING_BUFFER_BYTES]; + int inpOffset = 0; + + for(;;) + { + // Read random length ([1,MESSAGE_MAX_BYTES]) data to the ring buffer. + char* const inpPtr = &inpBuf[inpOffset]; + const int randomLength = (rand() % MESSAGE_MAX_BYTES) + 1; + const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength); + if (0 == inpBytes) break; + + { + char cmpBuf[LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)]; + const int cmpBytes = LZ4_compressHC_continue(lz4Stream, inpPtr, cmpBuf, inpBytes); + if(cmpBytes <= 0) break; + write_int32(outFp, cmpBytes); + write_bin(outFp, cmpBuf, cmpBytes); + + inpOffset += inpBytes; + + // Wraparound the ringbuffer offset + if(inpOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES) + inpOffset = 0; + } + } + + write_int32(outFp, 0); +} + + +void test_decompress(FILE* outFp, FILE* inpFp) +{ + static char decBuf[RING_BUFFER_BYTES]; + int decOffset = 0; + LZ4_streamDecode_t lz4StreamDecode_body = { 0 }; + LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body; + + for(;;) { + int cmpBytes = 0; + char cmpBuf[LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)]; + + { + const size_t r0 = read_int32(inpFp, &cmpBytes); + size_t r1; + if(r0 != 1 || cmpBytes <= 0) break; + + r1 = read_bin(inpFp, cmpBuf, cmpBytes); + if(r1 != (size_t) cmpBytes) break; + } + + { + char* const decPtr = &decBuf[decOffset]; + const int decBytes = LZ4_decompress_safe_continue( + lz4StreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES); + if(decBytes <= 0) + break; + decOffset += decBytes; + write_bin(outFp, decPtr, decBytes); + + // Wraparound the ringbuffer offset + if(decOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES) decOffset = 0; + } + } +} + + +int compare(FILE* f0, FILE* f1) +{ + int result = 0; + + while(0 == result) { + char b0[65536]; + char b1[65536]; + const size_t r0 = fread(b0, 1, sizeof(b0), f0); + const size_t r1 = fread(b1, 1, sizeof(b1), f1); + + result = (int) r0 - (int) r1; + + if(0 == r0 || 0 == r1) { + break; + } + if(0 == result) { + result = memcmp(b0, b1, r0); + } + } + + return result; +} + + +int main(int argc, char** argv) +{ + char inpFilename[256] = { 0 }; + char lz4Filename[256] = { 0 }; + char decFilename[256] = { 0 }; + + if(argc < 2) { + printf("Please specify input filename\n"); + return 0; + } + + snprintf(inpFilename, 256, "%s", argv[1]); + snprintf(lz4Filename, 256, "%s.lz4s-%d", argv[1], 0); + snprintf(decFilename, 256, "%s.lz4s-%d.dec", argv[1], 0); + + printf("inp = [%s]\n", inpFilename); + printf("lz4 = [%s]\n", lz4Filename); + printf("dec = [%s]\n", decFilename); + + // compress + { + FILE* inpFp = fopen(inpFilename, "rb"); + FILE* outFp = fopen(lz4Filename, "wb"); + + test_compress(outFp, inpFp); + + fclose(outFp); + fclose(inpFp); + } + + // decompress + { + FILE* inpFp = fopen(lz4Filename, "rb"); + FILE* outFp = fopen(decFilename, "wb"); + + test_decompress(outFp, inpFp); + + fclose(outFp); + fclose(inpFp); + } + + // verify + { + FILE* inpFp = fopen(inpFilename, "rb"); + FILE* decFp = fopen(decFilename, "rb"); + + const int cmp = compare(inpFp, decFp); + if(0 == cmp) { + printf("Verify : OK\n"); + } else { + printf("Verify : NG\n"); + } + + fclose(decFp); + fclose(inpFp); + } + + return 0; +} diff --git a/examples/Makefile b/examples/Makefile index 5704c55..b5f9a7e 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -58,7 +58,7 @@ endif default: all -all: printVersion doubleBuffer ringBuffer lineCompress +all: printVersion doubleBuffer ringBuffer ringBufferHC lineCompress printVersion: $(LZ4DIR)/lz4.c printVersion.c $(CC) $(FLAGS) $^ -o $@$(EXT) @@ -69,6 +69,9 @@ doubleBuffer: $(LZ4DIR)/lz4.c blockStreaming_doubleBuffer.c ringBuffer : $(LZ4DIR)/lz4.c blockStreaming_ringBuffer.c $(CC) $(FLAGS) $^ -o $@$(EXT) +ringBufferHC: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c HCStreaming_ringBuffer.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + lineCompress: $(LZ4DIR)/lz4.c blockStreaming_lineByLine.c $(CC) $(FLAGS) $^ -o $@$(EXT) @@ -76,10 +79,11 @@ test : all ./printVersion$(EXT) ./doubleBuffer$(EXT) $(TESTFILE) ./ringBuffer$(EXT) $(TESTFILE) + ./ringBufferHC$(EXT) $(TESTFILE) ./lineCompress$(EXT) $(TESTFILE) clean: @rm -f core *.o *.dec *-0 *-8192 *.lz4s \ - printVersion$(EXT) doubleBuffer$(EXT) ringBuffer$(EXT) lineCompress$(EXT) + printVersion$(EXT) doubleBuffer$(EXT) ringBuffer$(EXT) ringBufferHC$(EXT) lineCompress$(EXT) @echo Cleaning completed diff --git a/lz4hc.c b/lz4hc.c index 38e0ce4..ce273de 100755 --- a/lz4hc.c +++ b/lz4hc.c @@ -379,6 +379,21 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip) } +static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock) +{ + if (ctxPtr->end >= ctxPtr->base + 4) + LZ4HC_Insert (ctxPtr, ctxPtr->end-3); // finish referencing dictionary content + // Note : need to handle risk of index overflow + // Use only one memory segment for dict, so any previous External Dict is lost at this stage + ctxPtr->lowLimit = ctxPtr->dictLimit; + ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); + ctxPtr->dictBase = ctxPtr->base; + ctxPtr->base = newBlock - ctxPtr->dictLimit; + ctxPtr->end = newBlock; + ctxPtr->nextToUpdate = ctxPtr->dictLimit; // reference table must skip to from beginning of block +} + + static size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* const p1Limit) { const BYTE* const p1Start = p1; @@ -397,21 +412,6 @@ static size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* co } -static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock) -{ - if (ctxPtr->end >= ctxPtr->base + 4) - LZ4HC_Insert (ctxPtr, ctxPtr->end-3); // finish referencing dictionary content - // Note : need to handle risk of index overflow - // Use only one memory segment for dict, so any previous External Dict is lost at this stage - ctxPtr->lowLimit = ctxPtr->dictLimit; - ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); - ctxPtr->dictBase = ctxPtr->base; - ctxPtr->base = newBlock - ctxPtr->dictLimit; - ctxPtr->end = newBlock; - ctxPtr->nextToUpdate = ctxPtr->dictLimit; // reference table must skip to from beginning of block -} - - FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, // Index table will be updated const BYTE* ip, const BYTE* const iLimit, const BYTE** matchpos, @@ -618,8 +618,6 @@ static int LZ4HC_compress_generic ( if (compressionLevel > MAX_COMPRESSION_LEVEL) compressionLevel = MAX_COMPRESSION_LEVEL; if (compressionLevel == 0) compressionLevel = LZ4HC_DEFAULT_COMPRESSIONLEVEL; maxNbAttempts = 1 << compressionLevel; - /* check if blocks follow each other */ - if (ip != ctx->end) LZ4HC_setExternalDict(ctx, ip); ctx->end += inputSize; ip++; @@ -866,16 +864,46 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int /* compression */ +static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* dsPtr, + const char* source, char* dest, + int inputSize, int maxOutputSize, limitedOutput_directive limit) +{ + /* auto-init if forgotten */ + if (dsPtr->base == NULL) + LZ4HC_init (dsPtr, (const BYTE*) source); + + /* check if blocks follow each other */ + if ((const BYTE*)source != dsPtr->end) LZ4HC_setExternalDict(dsPtr, (const BYTE*)source); + + /* Check overlapping input/dictionary space */ + { + const BYTE* sourceEnd = (const BYTE*) source + inputSize; + const BYTE* dictBegin = dsPtr->dictBase + dsPtr->lowLimit; + const BYTE* dictEnd = dsPtr->dictBase + dsPtr->dictLimit; + if ((sourceEnd > dictBegin) && (sourceEnd < dictEnd)) + { + dsPtr->lowLimit = (U32)(sourceEnd - dsPtr->dictBase); + if (dsPtr->dictLimit - dsPtr->lowLimit < 4) dsPtr->lowLimit = dsPtr->dictLimit; + } + } + + return LZ4HC_compress_generic (dsPtr, source, dest, inputSize, maxOutputSize, dsPtr->compressionLevel, limit); +} + int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize) { - if (((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base == NULL) LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) source); - return LZ4HC_compress_generic (LZ4_streamHCPtr, source, dest, inputSize, 0, ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel, noLimit); + //if (((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base == NULL) + // LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) source); + //return LZ4HC_compress_generic (LZ4_streamHCPtr, source, dest, inputSize, 0, ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel, noLimit); + return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, 0, noLimit); } int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize) { - if (((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base == NULL) LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) source); - return LZ4HC_compress_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel, limitedOutput); + //if (((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base == NULL) + // LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) source); + //return LZ4HC_compress_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel, limitedOutput); + return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput); } -- cgit v0.12 From 2b421e97d4e7cfbefdc007bf30133b0de7e7e14e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 25 Oct 2014 20:52:10 +0100 Subject: HC streaming : support small ringbuffer scenarios --- examples/HCStreaming_ringBuffer.c | 90 ++++++--- examples/Makefile | 2 +- lz4.c | 145 ++++++++------ lz4.h | 30 +-- lz4hc.c | 13 +- lz4hc.h | 5 +- programs/Makefile | 2 +- programs/fuzzer.c | 411 ++++++++++++++++++++++++-------------- 8 files changed, 435 insertions(+), 263 deletions(-) mode change 100755 => 100644 lz4hc.c mode change 100755 => 100644 lz4hc.h mode change 100644 => 100755 programs/Makefile diff --git a/examples/HCStreaming_ringBuffer.c b/examples/HCStreaming_ringBuffer.c index 3532fa3..53e14ca 100755 --- a/examples/HCStreaming_ringBuffer.c +++ b/examples/HCStreaming_ringBuffer.c @@ -3,7 +3,7 @@ /************************************** - Compiler Options +Compiler Options **************************************/ #ifdef _MSC_VER /* Visual Studio */ # define _CRT_SECURE_NO_WARNINGS // for MSVC @@ -18,7 +18,7 @@ /************************************** - Includes +Includes **************************************/ #include "lz4hc.h" #include "lz4.h" @@ -30,8 +30,8 @@ enum { MESSAGE_MAX_BYTES = 1024, - RING_BUFFER_BYTES = 1024 * 256 + MESSAGE_MAX_BYTES, - DICT_BYTES = 65536, + RING_BUFFER_BYTES = 1024 * 8 + MESSAGE_MAX_BYTES, + DEC_BUFFER_BYTES = RING_BUFFER_BYTES + MESSAGE_MAX_BYTES }; @@ -59,9 +59,10 @@ void test_compress(FILE* outFp, FILE* inpFp) static char inpBuf[RING_BUFFER_BYTES]; int inpOffset = 0; + unsigned done = 0; for(;;) - { + { // Read random length ([1,MESSAGE_MAX_BYTES]) data to the ring buffer. char* const inpPtr = &inpBuf[inpOffset]; const int randomLength = (rand() % MESSAGE_MAX_BYTES) + 1; @@ -71,11 +72,13 @@ void test_compress(FILE* outFp, FILE* inpFp) { char cmpBuf[LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)]; const int cmpBytes = LZ4_compressHC_continue(lz4Stream, inpPtr, cmpBuf, inpBytes); + if(cmpBytes <= 0) break; write_int32(outFp, cmpBytes); write_bin(outFp, cmpBuf, cmpBytes); inpOffset += inpBytes; + done += inpBytes; // Wraparound the ringbuffer offset if(inpOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES) @@ -89,22 +92,26 @@ void test_compress(FILE* outFp, FILE* inpFp) void test_decompress(FILE* outFp, FILE* inpFp) { - static char decBuf[RING_BUFFER_BYTES]; + static char decBuf[DEC_BUFFER_BYTES]; int decOffset = 0; LZ4_streamDecode_t lz4StreamDecode_body = { 0 }; LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body; + unsigned done = 0; - for(;;) { - int cmpBytes = 0; + for(;;) + { + int cmpBytes = 0; char cmpBuf[LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)]; { const size_t r0 = read_int32(inpFp, &cmpBytes); - size_t r1; - if(r0 != 1 || cmpBytes <= 0) break; + size_t r1; + if(r0 != 1 || cmpBytes <= 0) + break; r1 = read_bin(inpFp, cmpBuf, cmpBytes); - if(r1 != (size_t) cmpBytes) break; + if(r1 != (size_t) cmpBytes) + break; } { @@ -112,38 +119,56 @@ void test_decompress(FILE* outFp, FILE* inpFp) const int decBytes = LZ4_decompress_safe_continue( lz4StreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES); if(decBytes <= 0) - break; + break; + + done += decBytes; decOffset += decBytes; write_bin(outFp, decPtr, decBytes); // Wraparound the ringbuffer offset - if(decOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES) decOffset = 0; + if(decOffset >= DEC_BUFFER_BYTES - MESSAGE_MAX_BYTES) + decOffset = 0; } } } -int compare(FILE* f0, FILE* f1) +// Compare 2 files content +// return 0 if identical +// return ByteNb>0 if different +size_t compare(FILE* f0, FILE* f1) { - int result = 0; + size_t result = 1; - while(0 == result) { + for (;;) + { char b0[65536]; char b1[65536]; const size_t r0 = fread(b0, 1, sizeof(b0), f0); const size_t r1 = fread(b1, 1, sizeof(b1), f1); - result = (int) r0 - (int) r1; + if ((r0==0) && (r1==0)) return 0; // success - if(0 == r0 || 0 == r1) { - break; + if (r0 != r1) + { + size_t smallest = r0; + if (r1>ML_BITS)) == RUN_MASK) { @@ -931,8 +932,8 @@ FORCE_INLINE int LZ4_decompress_generic( length += s; } while (likely((endOnInput)?ip oend-LASTLITERALS)) goto _output_error; + if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */ - if (length <= (size_t)(dest-(char*)ref)) + if (length <= (size_t)(lowPrefix-match)) { - ref = dictEnd - (dest-(char*)ref); - memcpy(op, ref, length); + /* match can be copied as a single segment from external dictionary */ + match = dictEnd - (lowPrefix-match); + memcpy(op, match, length); op += length; } else { - size_t copySize = (size_t)(dest-(char*)ref); + /* match encompass external dictionary and current segment */ + size_t copySize = (size_t)(lowPrefix-match); memcpy(op, dictEnd - copySize, copySize); op += copySize; copySize = length - copySize; - if (copySize > (size_t)((char*)op-dest)) /* overlap */ + if (copySize > (size_t)(op-lowPrefix)) /* overlap within current segment */ { BYTE* const endOfMatch = op + copySize; - const BYTE* copyFrom = (BYTE*)dest; + const BYTE* copyFrom = lowPrefix; while (op < endOfMatch) *op++ = *copyFrom++; } else { - memcpy(op, dest, copySize); + memcpy(op, lowPrefix, copySize); op += copySize; } } @@ -1009,25 +1012,25 @@ FORCE_INLINE int LZ4_decompress_generic( /* copy repeated sequence */ cpy = op + length; - if (unlikely((op-ref)<(int)STEPSIZE)) + if (unlikely((op-match)<(int)STEPSIZE)) { - const size_t dec64 = dec64table[op-ref]; - op[0] = ref[0]; - op[1] = ref[1]; - op[2] = ref[2]; - op[3] = ref[3]; - ref += dec32table[op-ref]; - A32(op+4) = A32(ref); - op += 8; ref -= dec64; - } else { LZ4_COPY8(op,ref); } + const size_t dec64 = dec64table[op-match]; + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += dec32table[op-match]; + A32(op+4) = A32(match); + op += 8; match -= dec64; + } else { LZ4_COPY8(op,match); } if (unlikely(cpy>oend-12)) { if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last 5 bytes must be literals */ - if (opdictionary = dictionary; - lz4sd->dictSize = dictSize; + lz4sd->prefixSize = (size_t) dictSize; + lz4sd->prefixEnd = (BYTE*) dictionary + dictSize; + lz4sd->externalDict = NULL; + lz4sd->extDictSize = 0; return 1; } @@ -1104,23 +1111,32 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti These decoding functions allow decompression of multiple blocks in "streaming" mode. Previously decoded blocks must still be available at the memory position where they were decoded. If it's not possible, save the relevant part of decoded data into a safe buffer, - and indicate where it stands using LZ4_setDictDecode() + and indicate where it stands using LZ4_setStreamDecode() */ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) { LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; int result; - result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, lz4sd->dictionary, lz4sd->dictSize); - if (result <= 0) return result; - if (lz4sd->dictionary + lz4sd->dictSize == dest) + if (lz4sd->prefixEnd == (BYTE*)dest) { - lz4sd->dictSize += result; + result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, full, 0, + usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += result; + lz4sd->prefixEnd += result; } else { - lz4sd->dictionary = dest; - lz4sd->dictSize = result; + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; + result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, full, 0, + usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = result; + lz4sd->prefixEnd = (BYTE*)dest + result; } return result; @@ -1131,16 +1147,25 @@ int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; int result; - result = LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, usingExtDict, lz4sd->dictionary, lz4sd->dictSize); - if (result <= 0) return result; - if (lz4sd->dictionary + lz4sd->dictSize == dest) + if (lz4sd->prefixEnd == (BYTE*)dest) { - lz4sd->dictSize += result; + result = LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, full, 0, + usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += originalSize; + lz4sd->prefixEnd += originalSize; } else { - lz4sd->dictionary = dest; - lz4sd->dictSize = result; + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = (BYTE*)dest - lz4sd->extDictSize; + result = LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, full, 0, + usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = originalSize; + lz4sd->prefixEnd = (BYTE*)dest + originalSize; } return result; @@ -1157,28 +1182,26 @@ Advanced decoding functions : FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) { if (dictSize==0) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, NULL, 64 KB); + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 64 KB); if ((dictStart+dictSize == dest) && (dictSize >= (int)(64 KB - 1))) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, NULL, 64 KB); - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, dictStart, dictSize); + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 64 KB); + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (BYTE*)dictStart, dictSize); } int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) { - //return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, dictStart, dictSize); return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); } int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) { - //return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, usingExtDict, dictStart, dictSize); return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize); } /* debug function */ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) { - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, dictStart, dictSize); + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (BYTE*)dictStart, dictSize); } @@ -1258,10 +1281,10 @@ int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) { - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, NULL, 64 KB); + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); } int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) { - return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, NULL, 64 KB); + return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); } diff --git a/lz4.h b/lz4.h index 7ad736f..eb96067 100644 --- a/lz4.h +++ b/lz4.h @@ -47,8 +47,8 @@ extern "C" { Version **************************************/ #define LZ4_VERSION_MAJOR 1 /* for major interface/format changes */ -#define LZ4_VERSION_MINOR 3 /* for minor interface/format changes */ -#define LZ4_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */ +#define LZ4_VERSION_MINOR 4 /* for minor interface/format changes */ +#define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) int LZ4_versionNumber (void); @@ -234,7 +234,7 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_streamPtr, char* safeBuffer, int dictSize); Experimental Streaming Decompression Functions ************************************************/ -#define LZ4_STREAMDECODESIZE_U32 4 +#define LZ4_STREAMDECODESIZE_U32 8 #define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U32 * sizeof(unsigned int)) /* * LZ4_streamDecode_t @@ -244,29 +244,29 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_streamPtr, char* safeBuffer, int dictSize); typedef struct { unsigned int table[LZ4_STREAMDECODESIZE_U32]; } LZ4_streamDecode_t; /* + * If you prefer dynamic allocation methods, + * LZ4_createStreamDecode will allocate and initialize an LZ4_streamDecode_t structure + * LZ4_freeStreamDecode releases its memory. + */ +LZ4_streamDecode_t* LZ4_createStreamDecode(void); +int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); + +/* * LZ4_setStreamDecode * Use this function to instruct where to find the dictionary. * This function can be used to specify a static dictionary, * or to instruct where to find some previously decoded data saved into a different memory space. - * Setting a size of 0 is allowed (same effect as no dictionary). + * Setting a size of 0 is allowed (same effect as no dictionary, same effect as reset). * Return : 1 if OK, 0 if error */ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); /* - * If you prefer dynamic allocation methods, - * LZ4_createStreamDecode will allocate and initialize an LZ4_streamDecode_t structure - * LZ4_freeStreamDecode releases its memory. - */ -LZ4_streamDecode_t* LZ4_createStreamDecode(void); -int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); - -/* *_continue() : These decoding functions allow decompression of multiple blocks in "streaming" mode. - Previously decoded blocks must still be available at the memory position where they were decoded. - If it's not possible, save the relevant part of decoded data into a safe buffer, - and indicate where its new address using LZ4_setStreamDecode() + Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) + If this condition is not possible, save the relevant part of decoded data into a safe buffer, + and indicate where is its new address using LZ4_setStreamDecode() */ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); diff --git a/lz4hc.c b/lz4hc.c old mode 100755 new mode 100644 index ce273de..8fc9606 --- a/lz4hc.c +++ b/lz4hc.c @@ -546,14 +546,14 @@ FORCE_INLINE int LZ4HC_encodeSequence ( BYTE** op, const BYTE** anchor, int matchLength, - const BYTE* match, + const BYTE* const match, limitedOutput_directive limitedOutputBuffer, BYTE* oend) { int length; BYTE* token; - //printf("literal : %u -- match : %u \n", (U32)(*ip - *anchor), (U32)matchLength); // debug + //if (debug) printf("literal : %u -- match : %u -- offset : %u\n", (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match)); // debug /* Encode Literal length */ length = (int)(*ip - *anchor); @@ -880,8 +880,9 @@ static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* dsPtr, const BYTE* sourceEnd = (const BYTE*) source + inputSize; const BYTE* dictBegin = dsPtr->dictBase + dsPtr->lowLimit; const BYTE* dictEnd = dsPtr->dictBase + dsPtr->dictLimit; - if ((sourceEnd > dictBegin) && (sourceEnd < dictEnd)) + if ((sourceEnd > dictBegin) && ((BYTE*)source < dictEnd)) { + if (sourceEnd > dictEnd) sourceEnd = dictEnd; dsPtr->lowLimit = (U32)(sourceEnd - dsPtr->dictBase); if (dsPtr->dictLimit - dsPtr->lowLimit < 4) dsPtr->lowLimit = dsPtr->dictLimit; } @@ -892,17 +893,11 @@ static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* dsPtr, int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize) { - //if (((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base == NULL) - // LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) source); - //return LZ4HC_compress_generic (LZ4_streamHCPtr, source, dest, inputSize, 0, ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel, noLimit); return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, 0, noLimit); } int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize) { - //if (((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base == NULL) - // LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) source); - //return LZ4HC_compress_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel, limitedOutput); return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput); } diff --git a/lz4hc.h b/lz4hc.h old mode 100755 new mode 100644 index 72979b5..5fbf398 --- a/lz4hc.h +++ b/lz4hc.h @@ -135,6 +135,7 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int maxDi /* These functions compress data in successive blocks of any size, using previous blocks as dictionary. +One key assumption is that each previous block will remain read-accessible while compressing next block. Before starting compression, state must be properly initialized, using LZ4_resetStreamHC(). A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional). @@ -143,8 +144,8 @@ Then, use LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue() t They work like usual LZ4_compressHC() or LZ4_compressHC_limitedOutput(), but use previous memory blocks to improve compression. Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression. -If, for any reason, previous data block can't be preserved in memory for next block compression, -you can still preserve it by moving it to a safer place, +If, for any reason, previous data block can't be preserved in memory during next compression block, +you can save it to a safer memory space, using LZ4_saveDictHC(). */ diff --git a/programs/Makefile b/programs/Makefile old mode 100644 new mode 100755 index 04694f2..e7e57a1 --- a/programs/Makefile +++ b/programs/Makefile @@ -30,7 +30,7 @@ # fullbench32: Same as fullbench, but forced to compile in 32-bits mode # ########################################################################## -RELEASE=r123 +RELEASE=r124 DESTDIR?= PREFIX ?= /usr diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 225361b..07935fa 100755 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -1,29 +1,29 @@ /* - fuzzer.c - Fuzzer test tool for LZ4 - Copyright (C) Yann Collet 2012-2014 - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - - LZ4 source repository : http://code.google.com/p/lz4/ +fuzzer.c - Fuzzer test tool for LZ4 +Copyright (C) Yann Collet 2012-2014 +GPL v2 License + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +You can contact the author at : +- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html +- LZ4 source repository : http://code.google.com/p/lz4/ */ /************************************** - Remove Visual warning messages +Remove Visual warning messages **************************************/ #define _CRT_SECURE_NO_WARNINGS // fgets #ifdef _MSC_VER /* Visual Studio */ @@ -34,7 +34,7 @@ /************************************** - Includes +Includes **************************************/ #include #include // fgets, sscanf @@ -46,26 +46,26 @@ /************************************** - Basic Types +Basic Types **************************************/ #if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ # include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; +typedef uint8_t BYTE; +typedef uint16_t U16; +typedef uint32_t U32; +typedef int32_t S32; +typedef uint64_t U64; #else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; +typedef unsigned char BYTE; +typedef unsigned short U16; +typedef unsigned int U32; +typedef signed int S32; +typedef unsigned long long U64; #endif /************************************** - Constants +Constants **************************************/ #ifndef LZ4_VERSION # define LZ4_VERSION "" @@ -87,7 +87,7 @@ /***************************************** - Macros +Macros *****************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } @@ -97,7 +97,7 @@ static U32 g_time = 0; /********************************************************* - Fuzzer functions +Fuzzer functions *********************************************************/ static U32 FUZ_GetMilliStart(void) { @@ -178,88 +178,88 @@ static void FUZ_fillCompressibleNoiseBuffer(void* buffer, int bufferSize, double #define BLOCKSIZE_I134 (32 MB) static int FUZ_AddressOverflow(void) { - char* buffers[MAX_NB_BUFF_I134+1] = {0}; - int i, nbBuff=0; - int highAddress = 0; + char* buffers[MAX_NB_BUFF_I134+1] = {0}; + int i, nbBuff=0; + int highAddress = 0; - printf("Overflow tests : "); + printf("Overflow tests : "); - // Only possible in 32-bits - if (sizeof(void*)==8) - { - printf("64 bits mode : no overflow \n"); - fflush(stdout); - return 0; - } - - buffers[0] = (char*)malloc(BLOCKSIZE_I134); - buffers[1] = (char*)malloc(BLOCKSIZE_I134); - if ((!buffers[0]) || (!buffers[1])) - { - printf("not enough memory for tests \n"); - return 0; - } - for (nbBuff=2; nbBuff < MAX_NB_BUFF_I134; nbBuff++) - { - printf("%3i \b\b\b\b", nbBuff); - buffers[nbBuff] = (char*)malloc(BLOCKSIZE_I134); - //printf("%08X ", (U32)(size_t)(buffers[nbBuff])); - fflush(stdout); - - if (((size_t)buffers[nbBuff] > (size_t)0x80000000) && (!highAddress)) + // Only possible in 32-bits + if (sizeof(void*)==8) { - printf("high address detected : "); + printf("64 bits mode : no overflow \n"); fflush(stdout); - highAddress=1; + return 0; } - if (buffers[nbBuff]==NULL) goto _endOfTests; + buffers[0] = (char*)malloc(BLOCKSIZE_I134); + buffers[1] = (char*)malloc(BLOCKSIZE_I134); + if ((!buffers[0]) || (!buffers[1])) { - size_t sizeToGenerateOverflow = (size_t)(- ((size_t)buffers[nbBuff-1]) + 512); - int nbOf255 = (int)((sizeToGenerateOverflow / 255) + 1); - char* input = buffers[nbBuff-1]; - char* output = buffers[nbBuff]; - int r; - input[0] = (char)0xF0; // Literal length overflow - input[1] = (char)0xFF; - input[2] = (char)0xFF; - input[3] = (char)0xFF; - for(i = 4; i <= nbOf255+4; i++) input[i] = (char)0xff; - r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134); - if (r>0) goto _overflowError; - input[0] = (char)0x1F; // Match length overflow - input[1] = (char)0x01; - input[2] = (char)0x01; - input[3] = (char)0x00; - r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134); - if (r>0) goto _overflowError; - - output = buffers[nbBuff-2]; // Reverse in/out pointer order - input[0] = (char)0xF0; // Literal length overflow - input[1] = (char)0xFF; - input[2] = (char)0xFF; - input[3] = (char)0xFF; - r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134); - if (r>0) goto _overflowError; - input[0] = (char)0x1F; // Match length overflow - input[1] = (char)0x01; - input[2] = (char)0x01; - input[3] = (char)0x00; - r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134); - if (r>0) goto _overflowError; + printf("not enough memory for tests \n"); + return 0; } - } + for (nbBuff=2; nbBuff < MAX_NB_BUFF_I134; nbBuff++) + { + printf("%3i \b\b\b\b", nbBuff); + buffers[nbBuff] = (char*)malloc(BLOCKSIZE_I134); + //printf("%08X ", (U32)(size_t)(buffers[nbBuff])); + fflush(stdout); + + if (((size_t)buffers[nbBuff] > (size_t)0x80000000) && (!highAddress)) + { + printf("high address detected : "); + fflush(stdout); + highAddress=1; + } + if (buffers[nbBuff]==NULL) goto _endOfTests; - nbBuff++; + { + size_t sizeToGenerateOverflow = (size_t)(- ((size_t)buffers[nbBuff-1]) + 512); + int nbOf255 = (int)((sizeToGenerateOverflow / 255) + 1); + char* input = buffers[nbBuff-1]; + char* output = buffers[nbBuff]; + int r; + input[0] = (char)0xF0; // Literal length overflow + input[1] = (char)0xFF; + input[2] = (char)0xFF; + input[3] = (char)0xFF; + for(i = 4; i <= nbOf255+4; i++) input[i] = (char)0xff; + r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134); + if (r>0) goto _overflowError; + input[0] = (char)0x1F; // Match length overflow + input[1] = (char)0x01; + input[2] = (char)0x01; + input[3] = (char)0x00; + r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134); + if (r>0) goto _overflowError; + + output = buffers[nbBuff-2]; // Reverse in/out pointer order + input[0] = (char)0xF0; // Literal length overflow + input[1] = (char)0xFF; + input[2] = (char)0xFF; + input[3] = (char)0xFF; + r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134); + if (r>0) goto _overflowError; + input[0] = (char)0x1F; // Match length overflow + input[1] = (char)0x01; + input[2] = (char)0x01; + input[3] = (char)0x00; + r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134); + if (r>0) goto _overflowError; + } + } + + nbBuff++; _endOfTests: - for (i=0 ; i must fail - FUZ_DISPLAYTEST; + // Test HC compression with just one missing byte into output buffer => must fail + FUZ_DISPLAYTEST; compressedBuffer[HCcompressedSize-1] = 0; ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize-1); FUZ_CHECKTEST(ret, "LZ4_compressHC_limitedOutput should have failed (output buffer too small by 1 byte)"); FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-1], "LZ4_compressHC_limitedOutput overran output buffer") - /* Dictionary tests */ + /* Dictionary tests */ - // Compress using dictionary - FUZ_DISPLAYTEST; + // Compress using dictionary + FUZ_DISPLAYTEST; LZ4continue = LZ4_create (dict); - LZ4_compress_continue ((LZ4_stream_t*)LZ4continue, dict, compressedBuffer, dictSize); // Just to fill hash tables + LZ4_compress_continue ((LZ4_stream_t*)LZ4continue, dict, compressedBuffer, dictSize); // Just to fill hash tables blockContinueCompressedSize = LZ4_compress_continue ((LZ4_stream_t*)LZ4continue, block, compressedBuffer, blockSize); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed"); free (LZ4continue); @@ -557,7 +557,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size") - crcCheck = XXH32(decodedBuffer, blockSize, 0); + crcCheck = XXH32(decodedBuffer, blockSize, 0); if (crcCheck!=crcOrig) { int i=0; @@ -571,7 +571,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size") - crcCheck = XXH32(decodedBuffer, blockSize, 0); + crcCheck = XXH32(decodedBuffer, blockSize, 0); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); FUZ_DISPLAYTEST; @@ -599,30 +599,30 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do } // Compress HC using External dictionary - FUZ_DISPLAYTEST; - dict -= (FUZ_rand(&randState) & 7); // even bigger separation - if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; - LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); - blockContinueCompressedSize = LZ4_compressHC_continue(&LZ4dictHC, block, compressedBuffer, blockSize); - FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compressHC_continue failed"); + FUZ_DISPLAYTEST; + dict -= (FUZ_rand(&randState) & 7); // even bigger separation + if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; + LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); + blockContinueCompressedSize = LZ4_compressHC_continue(&LZ4dictHC, block, compressedBuffer, blockSize); + FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compressHC_continue failed"); - FUZ_DISPLAYTEST; - LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); - ret = LZ4_compressHC_limitedOutput_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); - FUZ_CHECKTEST(ret>0, "LZ4_compressHC_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer"); + FUZ_DISPLAYTEST; + LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); + ret = LZ4_compressHC_limitedOutput_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); + FUZ_CHECKTEST(ret>0, "LZ4_compressHC_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer"); - FUZ_DISPLAYTEST; - LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); - ret = LZ4_compressHC_limitedOutput_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize); - FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); - FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer"); + FUZ_DISPLAYTEST; + LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); + ret = LZ4_compressHC_limitedOutput_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize); + FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); + FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer"); FUZ_DISPLAYTEST; decodedBuffer[blockSize] = 0; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size") - crcCheck = XXH32(decodedBuffer, blockSize, 0); + crcCheck = XXH32(decodedBuffer, blockSize, 0); if (crcCheck!=crcOrig) { int i=0; @@ -632,8 +632,8 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); - // ***** End of tests *** // - // Fill stats + // ***** End of tests *** // + // Fill stats bytes += blockSize; cbytes += compressedSize; hcbytes += HCcompressedSize; @@ -648,7 +648,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do // unalloc { - int result = 0; + int result = 0; _exit: free(CNBuffer); free(compressedBuffer); @@ -663,8 +663,10 @@ _output_error: } } + #define testInputSize (128 KB) #define testCompressedSize (64 KB) +#define ringBufferSize (8 KB) static void FUZ_unitTests(void) { @@ -674,7 +676,8 @@ static void FUZ_unitTests(void) char testInput[testInputSize]; char testCompressed[testCompressedSize]; char testVerify[testInputSize]; - U32 randState = 0; + char ringBuffer[ringBufferSize]; + U32 randState = 1; // Init FUZ_fillCompressibleNoiseBuffer(testInput, testInputSize, 0.50, &randState); @@ -682,6 +685,75 @@ static void FUZ_unitTests(void) // 32-bits address space overflow test FUZ_AddressOverflow(); + // LZ4 steraming tests + { + LZ4_stream_t* statePtr; + LZ4_stream_t streamingState; + U64 crcOrig; + U64 crcNew; + int result; + + // Allocation test + statePtr = LZ4_createStream(); + FUZ_CHECKTEST(statePtr==NULL, "LZ4_createStream() allocation failed"); + LZ4_freeStream(statePtr); + + // simple compression test + crcOrig = XXH64(testInput, testCompressedSize, 0); + LZ4_resetStream(&streamingState); + result = LZ4_compress_limitedOutput_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1); + FUZ_CHECKTEST(result==0, "LZ4_compress_limitedOutput_continue() compression failed"); + + result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize); + FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); + crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); + + // ring buffer test + { + XXH64_state_t xxhOrig; + XXH64_state_t xxhNew; + LZ4_streamDecode_t decodeState; + const U32 maxMessageSizeLog = 10; + const U32 maxMessageSizeMask = (1< ringBufferSize) rNext = 0; + if (dNext + messageSize > dBufferSize) dNext = 0; + } + } + } + // LZ4 HC streaming tests { LZ4_streamHC_t* sp; @@ -780,12 +852,12 @@ static void FUZ_unitTests(void) FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %i failed", segNb); XXH64_update(&crcNewState, dst, segSize); crcNew = XXH64_digest(&crcNewState); - if (crcOrig!=crcNew) - { - size_t c=0; - while (dst[c] == testInput[segStart+c]) c++; - DISPLAY("Bad decompression at %u / %u \n", (U32)c, (U32)segSize); - } + if (crcOrig!=crcNew) + { + size_t c=0; + while (dst[c] == testInput[segStart+c]) c++; + DISPLAY("Bad decompression at %u / %u \n", (U32)c, (U32)segSize); + } FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() part %i corruption", segNb); dict = dst; @@ -799,6 +871,51 @@ static void FUZ_unitTests(void) segSize = (FUZ_rand(&randState) & 8191); } } + + // ring buffer test + { + XXH64_state_t xxhOrig; + XXH64_state_t xxhNew; + LZ4_streamDecode_t decodeState; + const U32 maxMessageSizeLog = 10; + const U32 maxMessageSizeMask = (1< ringBufferSize) rNext = 0; + if (dNext + messageSize > dBufferSize) dNext = 0; + } + } + } printf("All unit tests completed succesfully \n"); -- cgit v0.12 From 508855c48826aa0544a907f02b52515aabba5e16 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 26 Oct 2014 11:22:15 +0100 Subject: HC streaming : support for very long streaming (> 4 GB) scenarios --- lz4hc.c | 273 ++++++++++++++++++++++++++++-------------------------- programs/fuzzer.c | 60 +++++++++++- 2 files changed, 197 insertions(+), 136 deletions(-) diff --git a/lz4hc.c b/lz4hc.c index 8fc9606..6705199 100644 --- a/lz4hc.c +++ b/lz4hc.c @@ -1,46 +1,46 @@ /* - LZ4 HC - High Compression Mode of LZ4 - Copyright (C) 2011-2014, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - - LZ4 source repository : http://code.google.com/p/lz4/ +LZ4 HC - High Compression Mode of LZ4 +Copyright (C) 2011-2014, Yann Collet. +BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html +- LZ4 source repository : http://code.google.com/p/lz4/ */ /************************************** - Tuning Parameter +Tuning Parameter **************************************/ #define LZ4HC_DEFAULT_COMPRESSIONLEVEL 8 /************************************** - Memory routines +Memory routines **************************************/ #include /* calloc, free */ #define ALLOCATOR(s) calloc(1,s) @@ -50,25 +50,25 @@ /************************************** - CPU Feature Detection +CPU Feature Detection **************************************/ /* 32 or 64 bits ? */ #if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \ - || defined(__64BIT__) || defined(__mips64) \ - || defined(__powerpc64__) || defined(__powerpc64le__) \ - || defined(__ppc64__) || defined(__ppc64le__) \ - || defined(__PPC64__) || defined(__PPC64LE__) \ - || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) \ - || defined(__s390x__) ) /* Detects 64 bits mode */ + || defined(__64BIT__) || defined(__mips64) \ + || defined(__powerpc64__) || defined(__powerpc64le__) \ + || defined(__ppc64__) || defined(__ppc64le__) \ + || defined(__PPC64__) || defined(__PPC64LE__) \ + || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) \ + || defined(__s390x__) ) /* Detects 64 bits mode */ # define LZ4_ARCH64 1 #else # define LZ4_ARCH64 0 #endif /* - * Little Endian or Big Endian ? - * Overwrite the #define below if you know your architecture endianess - */ +* Little Endian or Big Endian ? +* Overwrite the #define below if you know your architecture endianess +*/ #include /* Apparently required to detect endianess */ #if defined (__GLIBC__) # include @@ -78,19 +78,19 @@ #elif (defined(__BIG_ENDIAN__) || defined(__BIG_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(__LITTLE_ENDIAN__) || defined(__LITTLE_ENDIAN) || defined(_LITTLE_ENDIAN)) # define LZ4_BIG_ENDIAN 1 #elif defined(__sparc) || defined(__sparc__) \ - || defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) \ - || defined(__hpux) || defined(__hppa) \ - || defined(_MIPSEB) || defined(__s390__) + || defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) \ + || defined(__hpux) || defined(__hppa) \ + || defined(_MIPSEB) || defined(__s390__) # define LZ4_BIG_ENDIAN 1 #else /* Little Endian assumed. PDP Endian and other very rare endian format are unsupported. */ #endif /* - * Unaligned memory access is automatically enabled for "common" CPU, such as x86. - * For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected - * If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance - */ +* Unaligned memory access is automatically enabled for "common" CPU, such as x86. +* For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected +* If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance +*/ #if defined(__ARM_FEATURE_UNALIGNED) # define LZ4_FORCE_UNALIGNED_ACCESS 1 #endif @@ -102,7 +102,7 @@ /************************************** - Compiler Options +Compiler Options **************************************/ #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ /* "restrict" is a known keyword */ @@ -138,28 +138,28 @@ /************************************** - Includes +Includes **************************************/ #include "lz4hc.h" #include "lz4.h" /************************************** - Basic Types +Basic Types **************************************/ #if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ # include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; +typedef uint8_t BYTE; +typedef uint16_t U16; +typedef uint32_t U32; +typedef int32_t S32; +typedef uint64_t U64; #else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; +typedef unsigned char BYTE; +typedef unsigned short U16; +typedef unsigned int U32; +typedef signed int S32; +typedef unsigned long long U64; #endif #if defined(__GNUC__) && !defined(LZ4_FORCE_UNALIGNED_ACCESS) @@ -190,7 +190,7 @@ typedef struct _U64_S { U64 v; } _PACKED U64_S; /************************************** - Constants +Constants **************************************/ #define MINMATCH 4 @@ -220,7 +220,7 @@ typedef struct _U64_S { U64 v; } _PACKED U64_S; /************************************** - Architecture-specific macros +Architecture-specific macros **************************************/ #if LZ4_ARCH64 /* 64-bit */ # define STEPSIZE 8 @@ -244,7 +244,7 @@ typedef struct _U64_S { U64 v; } _PACKED U64_S; /************************************** - Local Types + Local Types **************************************/ typedef struct { @@ -262,7 +262,7 @@ typedef struct /************************************** - Macros + Macros **************************************/ #define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(!!(c)) }; } /* Visual : use only *after* variable declarations */ #define LZ4_WILDCOPY(s,d,e) do { LZ4_COPYPACKET(s,d) } while (dend >= ctxPtr->base + 4) - LZ4HC_Insert (ctxPtr, ctxPtr->end-3); // finish referencing dictionary content + if (ctxPtr->end >= ctxPtr->base + 4) + LZ4HC_Insert (ctxPtr, ctxPtr->end-3); // finish referencing dictionary content // Note : need to handle risk of index overflow // Use only one memory segment for dict, so any previous External Dict is lost at this stage ctxPtr->lowLimit = ctxPtr->dictLimit; @@ -413,22 +413,22 @@ static size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* co FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, // Index table will be updated - const BYTE* ip, const BYTE* const iLimit, - const BYTE** matchpos, - const int maxNbAttempts) + const BYTE* ip, const BYTE* const iLimit, + const BYTE** matchpos, + const int maxNbAttempts) { U16* const chainTable = hc4->chainTable; U32* const HashTable = hc4->hashTable; const BYTE* const base = hc4->base; const BYTE* const dictBase = hc4->dictBase; const U32 dictLimit = hc4->dictLimit; - const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); + const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); U32 matchIndex; const BYTE* match; int nbAttempts=maxNbAttempts; size_t ml=0; - /* HC4 match finder */ + /* HC4 match finder */ LZ4HC_Insert(hc4, ip); matchIndex = HashTable[LZ4HC_hashPtr(ip)]; @@ -439,7 +439,7 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, // I { match = base + matchIndex; if (*(match+ml) == *(ip+ml) - && (A32(match) == A32(ip))) + && (A32(match) == A32(ip))) { size_t mlt = LZ4HC_CommonLength(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; if (mlt > ml) { ml = mlt; *matchpos = match; } @@ -456,7 +456,7 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, // I mlt = LZ4HC_CommonLength(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; if ((ip+mlt == vLimit) && (vLimit < iLimit)) mlt += LZ4HC_CommonLength(ip+mlt, base+dictLimit, iLimit); - if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } // virtual matchpos + if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } // virtual matchpos } } matchIndex -= chainTable[matchIndex & 0xFFFF]; @@ -467,20 +467,20 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, // I FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( - LZ4HC_Data_Structure* hc4, - const BYTE* ip, - const BYTE* iLowLimit, - const BYTE* iHighLimit, - int longest, - const BYTE** matchpos, - const BYTE** startpos, - const int maxNbAttempts) + LZ4HC_Data_Structure* hc4, + const BYTE* ip, + const BYTE* iLowLimit, + const BYTE* iHighLimit, + int longest, + const BYTE** matchpos, + const BYTE** startpos, + const int maxNbAttempts) { U16* const chainTable = hc4->chainTable; U32* const HashTable = hc4->hashTable; const BYTE* const base = hc4->base; const U32 dictLimit = hc4->dictLimit; - const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); + const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); const BYTE* const dictBase = hc4->dictBase; const BYTE* match; U32 matchIndex; @@ -488,7 +488,7 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( int delta = (int)(ip-iLowLimit); - /* First Match */ + /* First Match */ LZ4HC_Insert(hc4, ip); matchIndex = HashTable[LZ4HC_hashPtr(ip)]; @@ -499,21 +499,21 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( { match = base + matchIndex; if (*(iLowLimit + longest) == *(match - delta + longest)) - if (A32(match) == A32(ip)) - { - const BYTE* startt = ip; - const BYTE* tmpMatch = match; - const BYTE* const matchEnd = ip + MINMATCH + LZ4HC_CommonLength(ip+MINMATCH, match+MINMATCH, iHighLimit); + if (A32(match) == A32(ip)) + { + const BYTE* startt = ip; + const BYTE* tmpMatch = match; + const BYTE* const matchEnd = ip + MINMATCH + LZ4HC_CommonLength(ip+MINMATCH, match+MINMATCH, iHighLimit); - while ((startt>iLowLimit) && (tmpMatch > iLowLimit) && (startt[-1] == tmpMatch[-1])) {startt--; tmpMatch--;} + while ((startt>iLowLimit) && (tmpMatch > iLowLimit) && (startt[-1] == tmpMatch[-1])) {startt--; tmpMatch--;} - if ((matchEnd-startt) > longest) - { - longest = (int)(matchEnd-startt); - *matchpos = tmpMatch; - *startpos = startt; + if ((matchEnd-startt) > longest) + { + longest = (int)(matchEnd-startt); + *matchpos = tmpMatch; + *startpos = startt; + } } - } } else { @@ -521,14 +521,14 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( if (A32(match) == A32(ip)) { size_t mlt; - int back=0; + int back=0; const BYTE* vLimit = ip + (dictLimit - matchIndex); if (vLimit > iHighLimit) vLimit = iHighLimit; mlt = LZ4HC_CommonLength(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) mlt += LZ4HC_CommonLength(ip+mlt, base+dictLimit, iHighLimit); - while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == match[back-1])) back--; - mlt -= back; + while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == match[back-1])) back--; + mlt -= back; if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; } } } @@ -541,14 +541,16 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive; +//static unsigned debug = 0; + FORCE_INLINE int LZ4HC_encodeSequence ( - const BYTE** ip, - BYTE** op, - const BYTE** anchor, - int matchLength, - const BYTE* const match, - limitedOutput_directive limitedOutputBuffer, - BYTE* oend) + const BYTE** ip, + BYTE** op, + const BYTE** anchor, + int matchLength, + const BYTE* const match, + limitedOutput_directive limitedOutputBuffer, + BYTE* oend) { int length; BYTE* token; @@ -584,14 +586,14 @@ FORCE_INLINE int LZ4HC_encodeSequence ( #define MAX_COMPRESSION_LEVEL 16 static int LZ4HC_compress_generic ( - void* ctxvoid, - const char* source, - char* dest, - int inputSize, - int maxOutputSize, - int compressionLevel, - limitedOutput_directive limit - ) + void* ctxvoid, + const char* source, + char* dest, + int inputSize, + int maxOutputSize, + int compressionLevel, + limitedOutput_directive limit + ) { LZ4HC_Data_Structure* ctx = (LZ4HC_Data_Structure*) ctxvoid; const BYTE* ip = (const BYTE*) source; @@ -665,10 +667,10 @@ _Search2: _Search3: /* - * Currently we have : - * ml2 > ml1, and - * ip1+3 <= ip2 (usually < ip1+ml1) - */ + * Currently we have : + * ml2 > ml1, and + * ip1+3 <= ip2 (usually < ip1+ml1) + */ if ((start2 - ip) < OPTIMAL_ML) { int correction; @@ -736,9 +738,9 @@ _Search3: } /* - * OK, now we have 3 ascending matches; let's write at least the first one - * ip & ref are known; Now for ml - */ + * OK, now we have 3 ascending matches; let's write at least the first one + * ip & ref are known; Now for ml + */ if (start2 < ip+ml) { if ((start2 - ip) < (int)ML_MASK) @@ -810,7 +812,7 @@ int LZ4_compressHC_limitedOutput(const char* source, char* dest, int inputSize, /***************************** - Using external allocation + Using external allocation *****************************/ int LZ4_sizeofStateHC(void) { return sizeof(LZ4HC_Data_Structure); } @@ -838,7 +840,7 @@ int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, c /************************************** - Experimental Streaming Functions + Experimental Streaming Functions **************************************/ /* allocation */ LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); } @@ -856,7 +858,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) { LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) dictionary); - if (dictSize >= 4) LZ4HC_Insert ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*)dictionary +(dictSize-3)); + if (dictSize >= 4) LZ4HC_Insert ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*)dictionary +(dictSize-3)); ((LZ4HC_Data_Structure*) LZ4_streamHCPtr)->end = (const BYTE*)dictionary + dictSize; return 1; } @@ -872,10 +874,19 @@ static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* dsPtr, if (dsPtr->base == NULL) LZ4HC_init (dsPtr, (const BYTE*) source); - /* check if blocks follow each other */ + /* Check overflow */ + if ((size_t)(dsPtr->end - dsPtr->base) > 2 GB) + { + size_t dictSize = (size_t)(dsPtr->end - dsPtr->base) - dsPtr->dictLimit; + if (dictSize > 64 KB) dictSize = 64 KB; + + LZ4_loadDictHC((LZ4_streamHC_t*)dsPtr, (const char*)(dsPtr->end) - dictSize, (int)dictSize); + } + + /* Check if blocks follow each other */ if ((const BYTE*)source != dsPtr->end) LZ4HC_setExternalDict(dsPtr, (const BYTE*)source); - /* Check overlapping input/dictionary space */ + /* Check overlapping input/dictionary space */ { const BYTE* sourceEnd = (const BYTE*) source + inputSize; const BYTE* dictBegin = dsPtr->dictBase + dsPtr->lowLimit; @@ -918,7 +929,7 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS /*********************************** - Deprecated Streaming functions +Deprecated Streaming functions ***********************************/ int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; } @@ -945,11 +956,11 @@ int LZ4_freeHC (void* LZ4HC_Data) /* int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize) { - return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, 0, noLimit); +return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, 0, noLimit); } int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize) { - return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, 0, limitedOutput); +return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, 0, limitedOutput); } */ diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 07935fa..b7721c8 100755 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -97,7 +97,7 @@ static U32 g_time = 0; /********************************************************* -Fuzzer functions + Fuzzer functions *********************************************************/ static U32 FUZ_GetMilliStart(void) { @@ -108,7 +108,6 @@ static U32 FUZ_GetMilliStart(void) return nCount; } - static U32 FUZ_GetMilliSpan(U32 nTimeStart) { U32 nCurrent = FUZ_GetMilliStart(); @@ -118,7 +117,6 @@ static U32 FUZ_GetMilliSpan(U32 nTimeStart) return nSpan; } - static U32 FUZ_rotl32(U32 u32, U32 nbBits) { return ((u32 << nbBits) | (u32 >> (32 - nbBits))); @@ -685,7 +683,7 @@ static void FUZ_unitTests(void) // 32-bits address space overflow test FUZ_AddressOverflow(); - // LZ4 steraming tests + // LZ4 streaming tests { LZ4_stream_t* statePtr; LZ4_stream_t streamingState; @@ -916,9 +914,61 @@ static void FUZ_unitTests(void) } } + // long stream test ; Warning : very long test ! + if (1) + { + XXH64_state_t crcOrigState; + XXH64_state_t crcNewState; + const U64 totalTestSize = 6ULL << 30; + U64 totalTestDone = 0; + size_t oldStart = 0; + size_t oldSize = 0; + U32 segNb = 1; + + DISPLAY("Long HC streaming test (%u MB)\n", (U32)(totalTestSize >> 20)); + LZ4_resetStreamHC(&sHC, 0); + + XXH64_reset(&crcOrigState, 0); + XXH64_reset(&crcNewState, 0); + + while (totalTestDone < totalTestSize) + { + size_t testSize = (FUZ_rand(&randState) & 65535) + 1; + size_t testStart = FUZ_rand(&randState) & 65535; + + FUZ_displayUpdate((U32)(totalTestDone >> 20)); + + XXH64_update(&crcOrigState, testInput + testStart, testSize); + crcOrig = XXH64_digest(&crcOrigState); + + result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + testStart, testCompressed, (int)testSize, LZ4_compressBound((int)testSize)); + FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); + + result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, (int)testSize, testInput + oldStart, (int)oldSize); + FUZ_CHECKTEST(result!=(int)testSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %i failed", segNb); + + XXH64_update(&crcNewState, testVerify, testSize); + crcNew = XXH64_digest(&crcNewState); + if (crcOrig!=crcNew) + { + size_t c=0; + while (testVerify[c] == testInput[testStart+c]) c++; + DISPLAY("Bad decompression at %u / %u \n", (U32)c, (U32)testSize); + } + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() part %i corruption", segNb); + + oldStart = testStart; + oldSize = testSize; + totalTestDone += testSize; + + segNb ++; + } + + DISPLAY("\r"); + } } - printf("All unit tests completed succesfully \n"); + printf("All unit tests completed successfully \n"); return; _output_error: exit(1); -- cgit v0.12 From 7a677a77a683a6716307034a176228075ce9fea5 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 28 Oct 2014 00:39:06 +0100 Subject: LZ4 HC streaming integrated within lz4frame --- lz4.c | 10 +- lz4.h | 4 +- lz4frame.c | 843 +++++++++++++++++++++++++++------------------------ lz4frame.h | 2 +- lz4hc.c | 2 +- programs/Makefile | 8 +- programs/frametest.c | 15 +- programs/fuzzer.c | 0 8 files changed, 465 insertions(+), 419 deletions(-) mode change 100755 => 100644 lz4.c mode change 100755 => 100644 lz4frame.c mode change 100755 => 100644 programs/Makefile mode change 100755 => 100644 programs/frametest.c mode change 100755 => 100644 programs/fuzzer.c diff --git a/lz4.c b/lz4.c old mode 100755 new mode 100644 index e7f45d6..e8693ae --- a/lz4.c +++ b/lz4.c @@ -1182,9 +1182,13 @@ Advanced decoding functions : FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) { if (dictSize==0) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 64 KB); - if ((dictStart+dictSize == dest) && (dictSize >= (int)(64 KB - 1))) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 64 KB); + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0); + if (dictStart+dictSize == dest) + { + if (dictSize >= (int)(64 KB - 1)) + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0); + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); + } return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (BYTE*)dictStart, dictSize); } diff --git a/lz4.h b/lz4.h index eb96067..0a8a772 100644 --- a/lz4.h +++ b/lz4.h @@ -47,8 +47,8 @@ extern "C" { Version **************************************/ #define LZ4_VERSION_MAJOR 1 /* for major interface/format changes */ -#define LZ4_VERSION_MINOR 4 /* for minor interface/format changes */ -#define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ +#define LZ4_VERSION_MINOR 3 /* for minor interface/format changes */ +#define LZ4_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) int LZ4_versionNumber (void); diff --git a/lz4frame.c b/lz4frame.c old mode 100755 new mode 100644 index f7bf242..aa46152 --- a/lz4frame.c +++ b/lz4frame.c @@ -1,45 +1,44 @@ /* - LZ4 auto-framing library - Copyright (C) 2011-2014, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +LZ4 auto-framing library +Copyright (C) 2011-2014, Yann Collet. +BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- LZ4 source repository : http://code.google.com/p/lz4/ +- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ -/* LZ4F is a stand-alone API to create LZ4-compressed frames - * fully conformant to specification v1.4.1. - * All related operations, including memory management, are handled by the library. - * You don't need lz4.h when using lz4frame.h. - * */ +/* LZ4F is a stand-alone API to create LZ4-compressed Frames +* fully conformant to specification v1.4.1. +* All related operations, including memory management, are handled by the library. +* */ /************************************** - Compiler Options +Compiler Options **************************************/ #ifdef _MSC_VER /* Visual Studio */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ @@ -53,7 +52,7 @@ /************************************** - Memory routines +Memory routines **************************************/ #include /* malloc, calloc, free */ #define ALLOCATOR(s) calloc(1,s) @@ -63,7 +62,7 @@ /************************************** - Includes +Includes **************************************/ #include "lz4frame.h" #include "lz4.h" @@ -72,7 +71,7 @@ /************************************** - Basic Types +Basic Types **************************************/ #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ # include @@ -91,7 +90,7 @@ typedef unsigned long long U64; /************************************** - Constants +Constants **************************************/ #define KB *(1<<10) #define MB *(1<<20) @@ -106,24 +105,26 @@ typedef unsigned long long U64; #define LZ4F_MAGICNUMBER 0x184D2204U #define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U #define LZ4F_MAXHEADERFRAME_SIZE 7 -#define LZ4F_BLOCKSIZEID_DEFAULT 4 +#define LZ4F_BLOCKSIZEID_DEFAULT max64KB +static const U32 minHClevel = 3; /************************************** - Structures and local types +Structures and local types **************************************/ typedef struct { LZ4F_preferences_t prefs; - unsigned version; - unsigned cStage; + U32 version; + U32 cStage; size_t maxBlockSize; size_t maxBufferSize; BYTE* tmpBuff; BYTE* tmpIn; size_t tmpInSize; XXH32_state_t xxh; - LZ4_stream_t lz4ctx; + void* lz4CtxPtr; + U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */ } LZ4F_cctx_internal_t; typedef struct @@ -149,27 +150,16 @@ typedef struct /************************************** - Macros +Macros **************************************/ /************************************** - Error management +Error management **************************************/ #define LZ4F_GENERATE_STRING(STRING) #STRING, static const char* LZ4F_errorStrings[] = { LZ4F_LIST_ERRORS(LZ4F_GENERATE_STRING) }; -/* -typedef enum { OK_NoError=0, ERROR_GENERIC = 1, - ERROR_maxBlockSize_invalid, ERROR_blockMode_invalid, ERROR_contentChecksumFlag_invalid, - ERROR_compressionLevel_invalid, - ERROR_allocation_failed, - ERROR_srcSize_tooLarge, ERROR_dstMaxSize_tooSmall, - ERROR_decompressionFailed, - ERROR_checksum_invalid, - ERROR_maxCode - } LZ4F_errorCodes; error codes are negative unsigned values. - Compare function result to (-specificCode) */ int LZ4F_isError(LZ4F_errorCode_t code) { @@ -185,7 +175,7 @@ const char* LZ4F_getErrorName(LZ4F_errorCode_t code) /************************************** - Private functions +Private functions **************************************/ static size_t LZ4F_getBlockSize(unsigned blockSizeID) { @@ -225,7 +215,7 @@ static BYTE LZ4F_headerChecksum (const BYTE* header, size_t length) /************************************** - Simple compression functions +Simple compression functions **************************************/ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { @@ -258,14 +248,14 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere /* LZ4F_compressFrame() - * Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.4.1, in a single step. - * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. - * You can get the minimum value of dstMaxSize by using LZ4F_compressFrameBound() - * If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode) - * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. - * The result of the function is the number of bytes written into dstBuffer. - * The function outputs an error code if it fails (can be tested using LZ4F_isError()) - */ +* Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.4.1, in a single step. +* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. +* You can get the minimum value of dstMaxSize by using LZ4F_compressFrameBound() +* If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode) +* The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. +* The result of the function is the number of bytes written into dstBuffer. +* The function outputs an error code if it fails (can be tested using LZ4F_isError()) +*/ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { LZ4F_cctx_internal_t cctxI = { 0 }; /* works because no allocation */ @@ -317,28 +307,30 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf if (LZ4F_isError(errorCode)) return errorCode; dstPtr += errorCode; + FREEMEM(cctxI.lz4CtxPtr); + return (dstPtr - dstStart); } /*********************************** - * Advanced compression functions - * *********************************/ +* Advanced compression functions +* *********************************/ /* LZ4F_createCompressionContext() : - * The first thing to do is to create a compressionContext object, which will be used in all compression operations. - * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. - * The version provided MUST be LZ4F_VERSION. It is intended to track potential version differences between different binaries. - * The function will provide a pointer to an allocated LZ4F_compressionContext_t object. - * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. - * Object can release its memory using LZ4F_freeCompressionContext(); - */ +* The first thing to do is to create a compressionContext object, which will be used in all compression operations. +* This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. +* The version provided MUST be LZ4F_VERSION. It is intended to track potential version differences between different binaries. +* The function will provide a pointer to an allocated LZ4F_compressionContext_t object. +* If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. +* Object can release its memory using LZ4F_freeCompressionContext(); +*/ LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version) { LZ4F_cctx_internal_t* cctxPtr; - cctxPtr = ALLOCATOR(sizeof(LZ4F_cctx_internal_t)); - if (cctxPtr==NULL) return (LZ4F_errorCode_t)-ERROR_allocation_failed; + cctxPtr = (LZ4F_cctx_internal_t*)ALLOCATOR(sizeof(LZ4F_cctx_internal_t)); + if (cctxPtr==NULL) return (LZ4F_errorCode_t)(-ERROR_allocation_failed); cctxPtr->version = version; cctxPtr->cStage = 0; /* Next stage : write header */ @@ -353,6 +345,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp { LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)LZ4F_compressionContext; + FREEMEM(cctxPtr->lz4CtxPtr); FREEMEM(cctxPtr->tmpBuff); FREEMEM(LZ4F_compressionContext); @@ -361,11 +354,11 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp /* LZ4F_compressBegin() : - * will write the frame header into dstBuffer. - * dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is LZ4F_MAXHEADERFRAME_SIZE bytes. - * The result of the function is the number of bytes written into dstBuffer for the header - * or an error code (can be tested using LZ4F_isError()) - */ +* will write the frame header into dstBuffer. +* dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is LZ4F_MAXHEADERFRAME_SIZE bytes. +* The result of the function is the number of bytes written into dstBuffer for the header +* or an error code (can be tested using LZ4F_isError()) +*/ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_preferences_t* preferencesPtr) { LZ4F_preferences_t prefNull = { 0 }; @@ -378,9 +371,23 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds if (dstMaxSize < LZ4F_MAXHEADERFRAME_SIZE) return (size_t)-ERROR_dstMaxSize_tooSmall; if (cctxPtr->cStage != 0) return (size_t)-ERROR_GENERIC; if (preferencesPtr == NULL) preferencesPtr = &prefNull; + cctxPtr->prefs = *preferencesPtr; + + /* ctx Management */ + { + U32 targetCtxLevel = cctxPtr->prefs.compressionLevellz4CtxLevel < targetCtxLevel) + { + FREEMEM(cctxPtr->lz4CtxPtr); + if (cctxPtr->prefs.compressionLevellz4CtxPtr = (void*)LZ4_createStream(); + else + cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC(); + cctxPtr->lz4CtxLevel = targetCtxLevel; + } + } /* Buffer Management */ - cctxPtr->prefs = *preferencesPtr; if (cctxPtr->prefs.frameInfo.blockSizeID == 0) cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; cctxPtr->maxBlockSize = LZ4F_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID); @@ -392,13 +399,16 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds { cctxPtr->maxBufferSize = requiredBuffSize; FREEMEM(cctxPtr->tmpBuff); - cctxPtr->tmpBuff = ALLOCATOR(requiredBuffSize); + cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize); if (cctxPtr->tmpBuff == NULL) return (size_t)-ERROR_allocation_failed; } cctxPtr->tmpIn = cctxPtr->tmpBuff; cctxPtr->tmpInSize = 0; XXH32_reset(&(cctxPtr->xxh), 0); - LZ4_resetStream(&(cctxPtr->lz4ctx)); + if (cctxPtr->prefs.compressionLevellz4CtxPtr)); + else + LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel); /* Magic Number */ LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER); @@ -406,10 +416,9 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds headerStart = dstPtr; /* FLG Byte */ - //cctxPtr->prefs.frameInfo.blockMode = 1; // <============ debug *dstPtr++ = ((1 & _2BITS) << 6) /* Version('01') */ - + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) /* Block mode */ - + (char)((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2); /* Stream checksum */ + + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) /* Block mode */ + + (char)((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2); /* Stream checksum */ /* BD Byte */ *dstPtr++ = (char)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4); /* CRC Byte */ @@ -422,9 +431,9 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds /* LZ4F_compressBound() : gives the size of Dst buffer given a srcSize to handle worst case situations. - * The LZ4F_frameInfo_t structure is optional : - * you can provide NULL as argument, all preferences will then be set to default. - * */ +* The LZ4F_frameInfo_t structure is optional : +* you can provide NULL as argument, all preferences will then be set to default. +* */ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { const LZ4F_preferences_t prefsNull = { 0 }; @@ -441,14 +450,14 @@ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesP } -typedef int (*compressFunc_t)(void*, const char*, char*, int, int); +typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level); -static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx) +static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx, int level) { /* compress one block */ BYTE* cSizePtr = (BYTE*)dst; U32 cSize; - cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1)); + cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1), level); LZ4F_writeLE32(cSizePtr, cSize); if (cSize == 0) /* compression failed */ { @@ -460,17 +469,53 @@ static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, com } +static int LZ4F_localLZ4_compress_limitedOutput_withState(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) +{ + (void) level; + return LZ4_compress_limitedOutput_withState(ctx, src, dst, srcSize, dstSize); +} + +static int LZ4F_localLZ4_compress_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) +{ + (void) level; + return LZ4_compress_limitedOutput_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstSize); +} + +static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) +{ + (void) level; + return LZ4_compressHC_limitedOutput_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstSize); +} + +static compressFunc_t LZ4F_selectCompression(blockMode_t blockMode, U32 level) +{ + if (level < minHClevel) + { + if (blockMode == blockIndependent) return LZ4F_localLZ4_compress_limitedOutput_withState; + return LZ4F_localLZ4_compress_limitedOutput_continue; + } + if (blockMode == blockIndependent) return LZ4_compressHC2_limitedOutput_withStateHC; + return LZ4F_localLZ4_compressHC_limitedOutput_continue; +} + +static int LZ4F_localSaveDict(LZ4F_cctx_internal_t* cctxPtr) +{ + if (cctxPtr->prefs.compressionLevel < minHClevel) + return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); + return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); +} + typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus; /* LZ4F_compressUpdate() - * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. - * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. - * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode) - * You can get the minimum value of dstMaxSize by using LZ4F_compressBound() - * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. - * The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered. - * The function outputs an error code if it fails (can be tested using LZ4F_isError()) - */ +* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. +* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. +* If this condition is not respected, LZ4F_compress() will fail (result is an errorCode) +* You can get the minimum value of dstMaxSize by using LZ4F_compressBound() +* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. +* The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered. +* The function outputs an error code if it fails (can be tested using LZ4F_isError()) +*/ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr) { LZ4F_compressOptions_t cOptionsNull = { 0 }; @@ -489,9 +534,7 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; /* select compression function */ - compress = (cctxPtr->prefs.frameInfo.blockMode == blockLinked) ? - (int(*)(void*,const char*,char*,int,int))LZ4_compress_limitedOutput_continue : - LZ4_compress_limitedOutput_withState; + compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); /* complete tmp buffer */ if (cctxPtr->tmpInSize > 0) /* some data already within tmp buffer */ @@ -512,7 +555,7 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy); srcPtr += sizeToCopy; - dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, &(cctxPtr->lz4ctx)); + dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); if (cctxPtr->prefs.frameInfo.blockMode==blockLinked) cctxPtr->tmpIn += blockSize; cctxPtr->tmpInSize = 0; @@ -523,7 +566,7 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d { /* compress full block */ lastBlockCompressed = fromSrcBuffer; - dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, blockSize, compress, &(cctxPtr->lz4ctx)); + dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); srcPtr += blockSize; } @@ -531,7 +574,7 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d { /* compress remaining input < blockSize */ lastBlockCompressed = fromSrcBuffer; - dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, &(cctxPtr->lz4ctx)); + dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); srcPtr = srcEnd; } @@ -544,17 +587,17 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d } else { - int realDictSize; - realDictSize = LZ4_saveDict (&(cctxPtr->lz4ctx), (char*)(cctxPtr->tmpBuff), 64 KB); + int realDictSize = LZ4F_localSaveDict(cctxPtr); if (realDictSize==0) return (size_t)-ERROR_GENERIC; cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; } } /* keep tmpIn within limits */ - if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) /* necessarily blockLinked && lastBlockCompressed==fromTmpBuffer */ + if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) /* necessarily blockLinked && lastBlockCompressed==fromTmpBuffer */ + && !(cctxPtr->prefs.autoFlush)) { - LZ4_saveDict (&(cctxPtr->lz4ctx), (char*)(cctxPtr->tmpBuff), 64 KB); + LZ4F_localSaveDict(cctxPtr); cctxPtr->tmpIn = cctxPtr->tmpBuff + 64 KB; } @@ -575,20 +618,20 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d /* LZ4F_flush() - * Should you need to create compressed data immediately, without waiting for a block to be filled, - * you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext. - * The result of the function is the number of bytes written into dstBuffer - * (it can be zero, this means there was no data left within compressionContext) - * The function outputs an error code if it fails (can be tested using LZ4F_isError()) - * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. - */ +* Should you need to create compressed data immediately, without waiting for a block to be filled, +* you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext. +* The result of the function is the number of bytes written into dstBuffer +* (it can be zero, this means there was no data left within compressionContext) +* The function outputs an error code if it fails (can be tested using LZ4F_isError()) +* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. +*/ size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) { LZ4F_compressOptions_t cOptionsNull = { 0 }; LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; - int (*compress)(void*, const char*, char*, int, int); + compressFunc_t compress; if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */ @@ -598,19 +641,17 @@ size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, (void)compressOptionsPtr; /* not yet useful */ /* select compression function */ - compress = (cctxPtr->prefs.frameInfo.blockMode == blockLinked) ? - (int(*)(void*,const char*,char*,int,int))LZ4_compress_limitedOutput_continue : - LZ4_compress_limitedOutput_withState; + compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); /* compress tmp buffer */ - dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, &(cctxPtr->lz4ctx)); + dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); if (cctxPtr->prefs.frameInfo.blockMode==blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize; cctxPtr->tmpInSize = 0; /* keep tmpIn within limits */ if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) /* necessarily blockLinked */ { - LZ4_saveDict (&(cctxPtr->lz4ctx), (char*)(cctxPtr->tmpBuff), 64 KB); + LZ4F_localSaveDict(cctxPtr); cctxPtr->tmpIn = cctxPtr->tmpBuff + 64 KB; } @@ -619,14 +660,14 @@ size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, /* LZ4F_compressEnd() - * When you want to properly finish the compressed frame, just call LZ4F_compressEnd(). - * It will flush whatever data remained within compressionContext (like LZ4_flush()) - * but also properly finalize the frame, with an endMark and a checksum. - * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) - * The function outputs an error code if it fails (can be tested using LZ4F_isError()) - * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. - * compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same. - */ +* When you want to properly finish the compressed frame, just call LZ4F_compressEnd(). +* It will flush whatever data remained within compressionContext (like LZ4_flush()) +* but also properly finalize the frame, with an endMark and a checksum. +* The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) +* The function outputs an error code if it fails (can be tested using LZ4F_isError()) +* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. +* compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same. +*/ size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) { LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; @@ -655,18 +696,18 @@ size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstB /*********************************** - * Decompression functions - * *********************************/ +* Decompression functions +* *********************************/ /* Resource management */ /* LZ4F_createDecompressionContext() : - * The first thing to do is to create a decompressionContext object, which will be used in all decompression operations. - * This is achieved using LZ4F_createDecompressionContext(). - * The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext object. - * If the result LZ4F_errorCode_t is not zero, there was an error during context creation. - * Object can release its memory using LZ4F_freeDecompressionContext(); - */ +* The first thing to do is to create a decompressionContext object, which will be used in all decompression operations. +* This is achieved using LZ4F_createDecompressionContext(). +* The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext object. +* If the result LZ4F_errorCode_t is not zero, there was an error during context creation. +* Object can release its memory using LZ4F_freeDecompressionContext(); +*/ LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_compressionContext_t* LZ4F_decompressionContextPtr, unsigned versionNumber) { LZ4F_dctx_internal_t* dctxPtr; @@ -763,24 +804,24 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPt typedef enum { dstage_getHeader=0, dstage_storeHeader, dstage_decodeHeader, - dstage_getCBlockSize, dstage_storeCBlockSize, dstage_decodeCBlockSize, - dstage_copyDirect, - dstage_getCBlock, dstage_storeCBlock, dstage_decodeCBlock, - dstage_decodeCBlock_intoDst, dstage_decodeCBlock_intoTmp, dstage_flushOut, - dstage_getSuffix, dstage_storeSuffix, dstage_checkSuffix - } dStage_t; + dstage_getCBlockSize, dstage_storeCBlockSize, dstage_decodeCBlockSize, + dstage_copyDirect, + dstage_getCBlock, dstage_storeCBlock, dstage_decodeCBlock, + dstage_decodeCBlock_intoDst, dstage_decodeCBlock_intoTmp, dstage_flushOut, + dstage_getSuffix, dstage_storeSuffix, dstage_checkSuffix +} dStage_t; /* LZ4F_getFrameInfo() - * This function decodes frame header information, such as blockSize. - * It is optional : you could start by calling directly LZ4F_decompress() instead. - * The objective is to extract header information without starting decompression, typically for allocation purposes. - * LZ4F_getFrameInfo() can also be used *after* starting decompression, on a valid LZ4F_decompressionContext_t. - * The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). - * You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr) - * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress, - * or an error code which can be tested using LZ4F_isError(). - */ +* This function decodes frame header information, such as blockSize. +* It is optional : you could start by calling directly LZ4F_decompress() instead. +* The objective is to extract header information without starting decompression, typically for allocation purposes. +* LZ4F_getFrameInfo() can also be used *after* starting decompression, on a valid LZ4F_decompressionContext_t. +* The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). +* You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr) +* The function result is an hint of the better srcSize to use for next call to LZ4F_decompress, +* or an error code which can be tested using LZ4F_isError(). +*/ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t decompressionContext, LZ4F_frameInfo_t* frameInfoPtr, const void* srcBuffer, size_t* srcSizePtr) { LZ4F_dctx_internal_t* dctxPtr = (LZ4F_dctx_internal_t*)decompressionContext; @@ -889,22 +930,22 @@ static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, s /* LZ4F_decompress() - * Call this function repetitively to regenerate data compressed within srcBuffer. - * The function will attempt to decode *srcSizePtr from srcBuffer, into dstBuffer of maximum size *dstSizePtr. - * - * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). - * - * The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). - * If the number of bytes read is < number of bytes provided, then the decompression operation is not complete. - * You will have to call it again, continuing from where it stopped. - * - * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress. - * Basically, it's the size of the current (or remaining) compressed block + header of next block. - * Respecting the hint provides some boost to performance, since it allows less buffer shuffling. - * Note that this is just a hint, you can always provide any srcSize you want. - * When a frame is fully decoded, the function result will be 0. - * If decompression failed, function result is an error code which can be tested using LZ4F_isError(). - */ +* Call this function repetitively to regenerate data compressed within srcBuffer. +* The function will attempt to decode *srcSizePtr from srcBuffer, into dstBuffer of maximum size *dstSizePtr. +* +* The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). +* +* The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). +* If the number of bytes read is < number of bytes provided, then the decompression operation is not complete. +* You will have to call it again, continuing from where it stopped. +* +* The function result is an hint of the better srcSize to use for next call to LZ4F_decompress. +* Basically, it's the size of the current (or remaining) compressed block + header of next block. +* Respecting the hint provides some boost to performance, since it allows less buffer shuffling. +* Note that this is just a hint, you can always provide any srcSize you want. +* When a frame is fully decoded, the function result will be 0. +* If decompression failed, function result is an error code which can be tested using LZ4F_isError(). +*/ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, void* dstBuffer, size_t* dstSizePtr, const void* srcBuffer, size_t* srcSizePtr, @@ -942,315 +983,315 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, { case dstage_getHeader: - { - if (srcEnd-srcPtr >= 7) { - selectedIn = srcPtr; - srcPtr += 7; - dctxPtr->dStage = dstage_decodeHeader; + if (srcEnd-srcPtr >= 7) + { + selectedIn = srcPtr; + srcPtr += 7; + dctxPtr->dStage = dstage_decodeHeader; + break; + } + dctxPtr->tmpInSize = 0; + dctxPtr->dStage = dstage_storeHeader; break; } - dctxPtr->tmpInSize = 0; - dctxPtr->dStage = dstage_storeHeader; - break; - } case dstage_storeHeader: - { - size_t sizeToCopy = 7 - dctxPtr->tmpInSize; - if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; - memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy); - dctxPtr->tmpInSize += sizeToCopy; - srcPtr += sizeToCopy; - if (dctxPtr->tmpInSize < 7) { - nextSrcSizeHint = (7 - dctxPtr->tmpInSize) + 4; - doAnotherStage = 0; /* no enough src, wait to get some more */ + size_t sizeToCopy = 7 - dctxPtr->tmpInSize; + if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; + memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + dctxPtr->tmpInSize += sizeToCopy; + srcPtr += sizeToCopy; + if (dctxPtr->tmpInSize < 7) + { + nextSrcSizeHint = (7 - dctxPtr->tmpInSize) + 4; + doAnotherStage = 0; /* no enough src, wait to get some more */ + break; + } + selectedIn = dctxPtr->header; + dctxPtr->dStage = dstage_decodeHeader; break; } - selectedIn = dctxPtr->header; - dctxPtr->dStage = dstage_decodeHeader; - break; - } case dstage_decodeHeader: - { - LZ4F_errorCode_t errorCode = LZ4F_decodeHeader(dctxPtr, selectedIn, 7); - if (LZ4F_isError(errorCode)) return errorCode; - dctxPtr->dStage = dstage_getCBlockSize; - break; - } + { + LZ4F_errorCode_t errorCode = LZ4F_decodeHeader(dctxPtr, selectedIn, 7); + if (LZ4F_isError(errorCode)) return errorCode; + dctxPtr->dStage = dstage_getCBlockSize; + break; + } case dstage_getCBlockSize: - { - if ((srcEnd - srcPtr) >= 4) { - selectedIn = srcPtr; - srcPtr += 4; - dctxPtr->dStage = dstage_decodeCBlockSize; + if ((srcEnd - srcPtr) >= 4) + { + selectedIn = srcPtr; + srcPtr += 4; + dctxPtr->dStage = dstage_decodeCBlockSize; + break; + } + /* not enough input to read cBlockSize field */ + dctxPtr->tmpInSize = 0; + dctxPtr->dStage = dstage_storeCBlockSize; break; } - /* not enough input to read cBlockSize field */ - dctxPtr->tmpInSize = 0; - dctxPtr->dStage = dstage_storeCBlockSize; - break; - } case dstage_storeCBlockSize: - { - size_t sizeToCopy = 4 - dctxPtr->tmpInSize; - if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; - memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); - srcPtr += sizeToCopy; - dctxPtr->tmpInSize += sizeToCopy; - if (dctxPtr->tmpInSize < 4) /* not enough input to get full cBlockSize; wait for more */ { - nextSrcSizeHint = 4 - dctxPtr->tmpInSize; - doAnotherStage=0; + size_t sizeToCopy = 4 - dctxPtr->tmpInSize; + if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; + memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + srcPtr += sizeToCopy; + dctxPtr->tmpInSize += sizeToCopy; + if (dctxPtr->tmpInSize < 4) /* not enough input to get full cBlockSize; wait for more */ + { + nextSrcSizeHint = 4 - dctxPtr->tmpInSize; + doAnotherStage=0; + break; + } + selectedIn = dctxPtr->tmpIn; + dctxPtr->dStage = dstage_decodeCBlockSize; break; } - selectedIn = dctxPtr->tmpIn; - dctxPtr->dStage = dstage_decodeCBlockSize; - break; - } case dstage_decodeCBlockSize: - { - size_t nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU; - if (nextCBlockSize==0) /* frameEnd signal, no more CBlock */ { - dctxPtr->dStage = dstage_getSuffix; - break; - } - if (nextCBlockSize > dctxPtr->maxBlockSize) return (size_t)-ERROR_GENERIC; /* invalid cBlockSize */ - dctxPtr->tmpInTarget = nextCBlockSize; - if (LZ4F_readLE32(selectedIn) & LZ4F_BLOCKUNCOMPRESSED_FLAG) - { - dctxPtr->dStage = dstage_copyDirect; + size_t nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU; + if (nextCBlockSize==0) /* frameEnd signal, no more CBlock */ + { + dctxPtr->dStage = dstage_getSuffix; + break; + } + if (nextCBlockSize > dctxPtr->maxBlockSize) return (size_t)-ERROR_GENERIC; /* invalid cBlockSize */ + dctxPtr->tmpInTarget = nextCBlockSize; + if (LZ4F_readLE32(selectedIn) & LZ4F_BLOCKUNCOMPRESSED_FLAG) + { + dctxPtr->dStage = dstage_copyDirect; + break; + } + dctxPtr->dStage = dstage_getCBlock; + if (dstPtr==dstEnd) + { + nextSrcSizeHint = nextCBlockSize + 4; + doAnotherStage = 0; + } break; } - dctxPtr->dStage = dstage_getCBlock; - if (dstPtr==dstEnd) - { - nextSrcSizeHint = nextCBlockSize + 4; - doAnotherStage = 0; - } - break; - } case dstage_copyDirect: /* uncompressed block */ - { - size_t sizeToCopy = dctxPtr->tmpInTarget; - if ((size_t)(srcEnd-srcPtr) < sizeToCopy) sizeToCopy = srcEnd - srcPtr; /* not enough input to read full block */ - if ((size_t)(dstEnd-dstPtr) < sizeToCopy) sizeToCopy = dstEnd - dstPtr; - memcpy(dstPtr, srcPtr, sizeToCopy); - if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), srcPtr, (U32)sizeToCopy); - - /* dictionary management */ - if (dctxPtr->frameInfo.blockMode==blockLinked) - LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 0); - - srcPtr += sizeToCopy; - dstPtr += sizeToCopy; - if (sizeToCopy == dctxPtr->tmpInTarget) /* all copied */ { - dctxPtr->dStage = dstage_getCBlockSize; + size_t sizeToCopy = dctxPtr->tmpInTarget; + if ((size_t)(srcEnd-srcPtr) < sizeToCopy) sizeToCopy = srcEnd - srcPtr; /* not enough input to read full block */ + if ((size_t)(dstEnd-dstPtr) < sizeToCopy) sizeToCopy = dstEnd - dstPtr; + memcpy(dstPtr, srcPtr, sizeToCopy); + if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), srcPtr, (U32)sizeToCopy); + + /* dictionary management */ + if (dctxPtr->frameInfo.blockMode==blockLinked) + LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 0); + + srcPtr += sizeToCopy; + dstPtr += sizeToCopy; + if (sizeToCopy == dctxPtr->tmpInTarget) /* all copied */ + { + dctxPtr->dStage = dstage_getCBlockSize; + break; + } + dctxPtr->tmpInTarget -= sizeToCopy; /* still need to copy more */ + nextSrcSizeHint = dctxPtr->tmpInTarget + 4; + doAnotherStage = 0; break; } - dctxPtr->tmpInTarget -= sizeToCopy; /* still need to copy more */ - nextSrcSizeHint = dctxPtr->tmpInTarget + 4; - doAnotherStage = 0; - break; - } case dstage_getCBlock: /* entry from dstage_decodeCBlockSize */ - { - if ((size_t)(srcEnd-srcPtr) < dctxPtr->tmpInTarget) { - dctxPtr->tmpInSize = 0; - dctxPtr->dStage = dstage_storeCBlock; + if ((size_t)(srcEnd-srcPtr) < dctxPtr->tmpInTarget) + { + dctxPtr->tmpInSize = 0; + dctxPtr->dStage = dstage_storeCBlock; + break; + } + selectedIn = srcPtr; + srcPtr += dctxPtr->tmpInTarget; + dctxPtr->dStage = dstage_decodeCBlock; break; } - selectedIn = srcPtr; - srcPtr += dctxPtr->tmpInTarget; - dctxPtr->dStage = dstage_decodeCBlock; - break; - } case dstage_storeCBlock: - { - size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; - if (sizeToCopy > (size_t)(srcEnd-srcPtr)) sizeToCopy = srcEnd-srcPtr; - memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); - dctxPtr->tmpInSize += sizeToCopy; - srcPtr += sizeToCopy; - if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) /* need more input */ { - nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + 4; - doAnotherStage=0; + size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; + if (sizeToCopy > (size_t)(srcEnd-srcPtr)) sizeToCopy = srcEnd-srcPtr; + memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + dctxPtr->tmpInSize += sizeToCopy; + srcPtr += sizeToCopy; + if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) /* need more input */ + { + nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + 4; + doAnotherStage=0; + break; + } + selectedIn = dctxPtr->tmpIn; + dctxPtr->dStage = dstage_decodeCBlock; break; } - selectedIn = dctxPtr->tmpIn; - dctxPtr->dStage = dstage_decodeCBlock; - break; - } case dstage_decodeCBlock: - { - if ((size_t)(dstEnd-dstPtr) < dctxPtr->maxBlockSize) /* not enough place into dst : decode into tmpOut */ - dctxPtr->dStage = dstage_decodeCBlock_intoTmp; - else - dctxPtr->dStage = dstage_decodeCBlock_intoDst; - break; - } + { + if ((size_t)(dstEnd-dstPtr) < dctxPtr->maxBlockSize) /* not enough place into dst : decode into tmpOut */ + dctxPtr->dStage = dstage_decodeCBlock_intoTmp; + else + dctxPtr->dStage = dstage_decodeCBlock_intoDst; + break; + } case dstage_decodeCBlock_intoDst: - { - int (*decoder)(const char*, char*, int, int, const char*, int); - int decodedSize; + { + int (*decoder)(const char*, char*, int, int, const char*, int); + int decodedSize; - if (dctxPtr->frameInfo.blockMode == blockLinked) - decoder = LZ4_decompress_safe_usingDict; - else - decoder = LZ4F_decompress_safe; + if (dctxPtr->frameInfo.blockMode == blockLinked) + decoder = LZ4_decompress_safe_usingDict; + else + decoder = LZ4F_decompress_safe; - decodedSize = decoder((const char*)selectedIn, (char*)dstPtr, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); - if (decodedSize < 0) return (size_t)-ERROR_GENERIC; /* decompression failed */ - if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize); + decodedSize = decoder((const char*)selectedIn, (char*)dstPtr, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); + if (decodedSize < 0) return (size_t)-ERROR_GENERIC; /* decompression failed */ + if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize); - /* dictionary management */ - if (dctxPtr->frameInfo.blockMode==blockLinked) - LZ4F_updateDict(dctxPtr, dstPtr, decodedSize, dstStart, 0); + /* dictionary management */ + if (dctxPtr->frameInfo.blockMode==blockLinked) + LZ4F_updateDict(dctxPtr, dstPtr, decodedSize, dstStart, 0); - dstPtr += decodedSize; - dctxPtr->dStage = dstage_getCBlockSize; - break; - } + dstPtr += decodedSize; + dctxPtr->dStage = dstage_getCBlockSize; + break; + } case dstage_decodeCBlock_intoTmp: - { - /* not enough place into dst : decode into tmpOut */ - int (*decoder)(const char*, char*, int, int, const char*, int); - int decodedSize; + { + /* not enough place into dst : decode into tmpOut */ + int (*decoder)(const char*, char*, int, int, const char*, int); + int decodedSize; - if (dctxPtr->frameInfo.blockMode == blockLinked) - decoder = LZ4_decompress_safe_usingDict; - else - decoder = LZ4F_decompress_safe; + if (dctxPtr->frameInfo.blockMode == blockLinked) + decoder = LZ4_decompress_safe_usingDict; + else + decoder = LZ4F_decompress_safe; - /* ensure enough place for tmpOut */ - if (dctxPtr->frameInfo.blockMode == blockLinked) - { - if (dctxPtr->dict == dctxPtr->tmpOutBuffer) + /* ensure enough place for tmpOut */ + if (dctxPtr->frameInfo.blockMode == blockLinked) { - if (dctxPtr->dictSize > 128 KB) + if (dctxPtr->dict == dctxPtr->tmpOutBuffer) { - memcpy(dctxPtr->dict, dctxPtr->dict + dctxPtr->dictSize - 64 KB, 64 KB); - dctxPtr->dictSize = 64 KB; + if (dctxPtr->dictSize > 128 KB) + { + memcpy(dctxPtr->dict, dctxPtr->dict + dctxPtr->dictSize - 64 KB, 64 KB); + dctxPtr->dictSize = 64 KB; + } + dctxPtr->tmpOut = dctxPtr->dict + dctxPtr->dictSize; + } + else /* dict not within tmp */ + { + size_t reservedDictSpace = dctxPtr->dictSize; + if (reservedDictSpace > 64 KB) reservedDictSpace = 64 KB; + dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + reservedDictSpace; } - dctxPtr->tmpOut = dctxPtr->dict + dctxPtr->dictSize; - } - else /* dict not within tmp */ - { - size_t reservedDictSpace = dctxPtr->dictSize; - if (reservedDictSpace > 64 KB) reservedDictSpace = 64 KB; - dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + reservedDictSpace; } - } - /* Decode */ - decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); - if (decodedSize < 0) return (size_t)-ERROR_decompressionFailed; /* decompression failed */ - if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize); - dctxPtr->tmpOutSize = decodedSize; - dctxPtr->tmpOutStart = 0; - dctxPtr->dStage = dstage_flushOut; - break; - } + /* Decode */ + decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); + if (decodedSize < 0) return (size_t)-ERROR_decompressionFailed; /* decompression failed */ + if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize); + dctxPtr->tmpOutSize = decodedSize; + dctxPtr->tmpOutStart = 0; + dctxPtr->dStage = dstage_flushOut; + break; + } case dstage_flushOut: /* flush decoded data from tmpOut to dstBuffer */ - { - size_t sizeToCopy = dctxPtr->tmpOutSize - dctxPtr->tmpOutStart; - if (sizeToCopy > (size_t)(dstEnd-dstPtr)) sizeToCopy = dstEnd-dstPtr; - memcpy(dstPtr, dctxPtr->tmpOut + dctxPtr->tmpOutStart, sizeToCopy); + { + size_t sizeToCopy = dctxPtr->tmpOutSize - dctxPtr->tmpOutStart; + if (sizeToCopy > (size_t)(dstEnd-dstPtr)) sizeToCopy = dstEnd-dstPtr; + memcpy(dstPtr, dctxPtr->tmpOut + dctxPtr->tmpOutStart, sizeToCopy); - /* dictionary management */ - if (dctxPtr->frameInfo.blockMode==blockLinked) - LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 1); + /* dictionary management */ + if (dctxPtr->frameInfo.blockMode==blockLinked) + LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 1); - dctxPtr->tmpOutStart += sizeToCopy; - dstPtr += sizeToCopy; + dctxPtr->tmpOutStart += sizeToCopy; + dstPtr += sizeToCopy; - /* end of flush ? */ - if (dctxPtr->tmpOutStart == dctxPtr->tmpOutSize) - { - dctxPtr->dStage = dstage_getCBlockSize; + /* end of flush ? */ + if (dctxPtr->tmpOutStart == dctxPtr->tmpOutSize) + { + dctxPtr->dStage = dstage_getCBlockSize; + break; + } + nextSrcSizeHint = 4; + doAnotherStage = 0; /* still some data to flush */ break; } - nextSrcSizeHint = 4; - doAnotherStage = 0; /* still some data to flush */ - break; - } case dstage_getSuffix: - { - size_t suffixSize = dctxPtr->frameInfo.contentChecksumFlag * 4; - if (suffixSize == 0) /* frame completed */ { - nextSrcSizeHint = 0; - dctxPtr->dStage = dstage_getHeader; - doAnotherStage = 0; + size_t suffixSize = dctxPtr->frameInfo.contentChecksumFlag * 4; + if (suffixSize == 0) /* frame completed */ + { + nextSrcSizeHint = 0; + dctxPtr->dStage = dstage_getHeader; + doAnotherStage = 0; + break; + } + if ((srcEnd - srcPtr) >= 4) /* CRC present */ + { + selectedIn = srcPtr; + srcPtr += 4; + dctxPtr->dStage = dstage_checkSuffix; + break; + } + dctxPtr->tmpInSize = 0; + dctxPtr->dStage = dstage_storeSuffix; break; } - if ((srcEnd - srcPtr) >= 4) /* CRC present */ + + case dstage_storeSuffix: { - selectedIn = srcPtr; - srcPtr += 4; + size_t sizeToCopy = 4 - dctxPtr->tmpInSize; + if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; + memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + srcPtr += sizeToCopy; + dctxPtr->tmpInSize += sizeToCopy; + if (dctxPtr->tmpInSize < 4) /* not enough input to read complete suffix */ + { + nextSrcSizeHint = 4 - dctxPtr->tmpInSize; + doAnotherStage=0; + break; + } + selectedIn = dctxPtr->tmpIn; dctxPtr->dStage = dstage_checkSuffix; break; } - dctxPtr->tmpInSize = 0; - dctxPtr->dStage = dstage_storeSuffix; - break; - } - case dstage_storeSuffix: - { - size_t sizeToCopy = 4 - dctxPtr->tmpInSize; - if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; - memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); - srcPtr += sizeToCopy; - dctxPtr->tmpInSize += sizeToCopy; - if (dctxPtr->tmpInSize < 4) /* not enough input to read complete suffix */ + case dstage_checkSuffix: { - nextSrcSizeHint = 4 - dctxPtr->tmpInSize; - doAnotherStage=0; + U32 readCRC = LZ4F_readLE32(selectedIn); + U32 resultCRC = XXH32_digest(&(dctxPtr->xxh)); + if (readCRC != resultCRC) return (size_t)-ERROR_checksum_invalid; + nextSrcSizeHint = 0; + dctxPtr->dStage = dstage_getHeader; + doAnotherStage = 0; break; } - selectedIn = dctxPtr->tmpIn; - dctxPtr->dStage = dstage_checkSuffix; - break; - } - - case dstage_checkSuffix: - { - U32 readCRC = LZ4F_readLE32(selectedIn); - U32 resultCRC = XXH32_digest(&(dctxPtr->xxh)); - if (readCRC != resultCRC) return (size_t)-ERROR_checksum_invalid; - nextSrcSizeHint = 0; - dctxPtr->dStage = dstage_getHeader; - doAnotherStage = 0; - break; - } } } /* preserve dictionary within tmp if necessary */ if ( (dctxPtr->frameInfo.blockMode==blockLinked) - &&(dctxPtr->dict != dctxPtr->tmpOutBuffer) - &&(!decompressOptionsPtr->stableDst) - &&((unsigned)(dctxPtr->dStage-1) < (unsigned)(dstage_getSuffix-1)) - ) + &&(dctxPtr->dict != dctxPtr->tmpOutBuffer) + &&(!decompressOptionsPtr->stableDst) + &&((unsigned)(dctxPtr->dStage-1) < (unsigned)(dstage_getSuffix-1)) + ) { if (dctxPtr->dStage == dstage_flushOut) { diff --git a/lz4frame.h b/lz4frame.h index bb7d688..d31203d 100644 --- a/lz4frame.h +++ b/lz4frame.h @@ -70,7 +70,7 @@ typedef size_t LZ4F_errorCode_t; ITEM(ERROR_maxCode) #define LZ4F_GENERATE_ENUM(ENUM) ENUM, -typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; /* enum is exposed, to let programmer detect & handle specific errors */ +typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; /* enum is exposed, to detect & handle specific errors; compare function result to -enum value */ int LZ4F_isError(LZ4F_errorCode_t code); /* Basically : code > -ERROR_maxCode */ const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /* return enum as string */ diff --git a/lz4hc.c b/lz4hc.c index 6705199..1422a7c 100644 --- a/lz4hc.c +++ b/lz4hc.c @@ -920,7 +920,7 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS LZ4HC_Data_Structure* sp = (LZ4HC_Data_Structure*)LZ4_streamHCPtr; if (dictSize > 64 KB) dictSize = 64 KB; if (dictSize < 0) dictSize = 0; - if (dictSize > (sp->end - sp->base)) dictSize = (int)(sp->end - sp->base); + if (dictSize > (sp->end - (sp->base + sp->lowLimit))) dictSize = (int)(sp->end - (sp->base + sp->lowLimit)); memcpy(safeBuffer, sp->end - dictSize, dictSize); LZ4_loadDictHC(LZ4_streamHCPtr, safeBuffer, dictSize); return dictSize; diff --git a/programs/Makefile b/programs/Makefile old mode 100755 new mode 100644 index e7e57a1..e313fc5 --- a/programs/Makefile +++ b/programs/Makefile @@ -174,13 +174,13 @@ test-frame: frametest test-mem: lz4 datagen frametest ./datagen -g16KB > tmp - valgrind ./lz4 -9 -BD -f tmp /dev/null + valgrind --leak-check=yes ./lz4 -9 -BD -f tmp /dev/null ./datagen -g16MB > tmp - valgrind ./lz4 -9 -B5D -f tmp /dev/null + valgrind --leak-check=yes ./lz4 -9 -B5D -f tmp /dev/null ./datagen -g256MB > tmp - valgrind ./lz4 -B4D -f tmp /dev/null + valgrind --leak-check=yes ./lz4 -B4D -f tmp /dev/null rm tmp - valgrind ./frametest -i100 + valgrind --leak-check=yes ./frametest -i100 test-mem32: lz4c32 datagen # unfortunately, valgrind doesn't seem to work with non-native binary. If someone knows how to do a valgrind-test on a 32-bits exe with a 64-bits system... diff --git a/programs/frametest.c b/programs/frametest.c old mode 100755 new mode 100644 index abf0cde..c7e6f45 --- a/programs/frametest.c +++ b/programs/frametest.c @@ -426,10 +426,11 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi LZ4F_preferences_t* prefsPtr = &prefs; (void)FUZ_rand(&coreRand); // update rand seed - prefs.frameInfo.blockMode = BMId; - prefs.frameInfo.blockSizeID = BSId; - prefs.frameInfo.contentChecksumFlag = CCflag; + prefs.frameInfo.blockMode = (blockMode_t)BMId; + prefs.frameInfo.blockSizeID = (blockSizeID_t)BSId; + prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)CCflag; prefs.autoFlush = autoflush; + prefs.compressionLevel = FUZ_rand(&randState) % 5; if ((FUZ_rand(&randState)&0xF) == 1) prefsPtr = NULL; DISPLAYUPDATE(2, "\r%5u ", testNb); @@ -444,7 +445,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi { const BYTE* ip = (const BYTE*)srcBuffer + srcStart; const BYTE* const iend = ip + srcSize; - BYTE* op = compressedBuffer; + BYTE* op = (BYTE*)compressedBuffer; BYTE* const oend = op + LZ4F_compressFrameBound(srcDataLength, NULL); unsigned maxBits = FUZ_highbit((U32)srcSize); result = LZ4F_compressBegin(cCtx, op, oend-op, prefsPtr); @@ -478,9 +479,9 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi } { - const BYTE* ip = compressedBuffer; + const BYTE* ip = (const BYTE*)compressedBuffer; const BYTE* const iend = ip + cSize; - BYTE* op = decodedBuffer; + BYTE* op = (BYTE*)decodedBuffer; BYTE* const oend = op + srcDataLength; unsigned maxBits = FUZ_highbit((U32)cSize); unsigned nonContiguousDst = (FUZ_rand(&randState) & 3) == 1; @@ -602,7 +603,7 @@ int main(int argc, char** argv) pause = 1; break; - case 'i': + case 'i': argument++; nbTests=0; while ((*argument>='0') && (*argument<='9')) diff --git a/programs/fuzzer.c b/programs/fuzzer.c old mode 100755 new mode 100644 -- cgit v0.12 From 6436eca4faeb3ffa1c17179d24315634e5342e6d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 28 Oct 2014 11:32:42 +0100 Subject: Added : frametest32 --- programs/Makefile | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index e313fc5..00af2f9 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -94,6 +94,9 @@ fuzzer32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c fuzzer.c frametest: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c frametest.c $(CC) $(FLAGS) $^ -o $@$(EXT) +frametest32: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c frametest.c + $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) + datagen : datagen.c $(CC) $(FLAGS) $^ -o $@$(EXT) @@ -103,7 +106,8 @@ clean: lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) \ fullbench$(EXT) fullbench32$(EXT) \ fuzzer$(EXT) fuzzer32$(EXT) \ - frametest$(EXT) datagen$(EXT) + frametest$(EXT) frametest32$(EXT) \ + datagen$(EXT) @echo Cleaning completed @@ -134,7 +138,7 @@ uninstall: test-native: test-lz4 test-lz4c test-frame test-fullbench test-fuzzer test-mem -test-force32: test-lz4c32 test-fullbench32 test-fuzzer32 test-mem32 +test-force32: test-lz4c32 test-frame32 test-fullbench32 test-fuzzer32 test-mem32 test-all: test-native test-force32 @@ -172,6 +176,9 @@ test-fuzzer32: fuzzer32 test-frame: frametest ./frametest +test-frame32: frametest32 + ./frametest32 + test-mem: lz4 datagen frametest ./datagen -g16KB > tmp valgrind --leak-check=yes ./lz4 -9 -BD -f tmp /dev/null -- cgit v0.12 From 9bf331afc6e6984406d9b3338e33edfd9439fc69 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 28 Oct 2014 15:35:43 +0100 Subject: new Travis CI build matrix --- .travis.yml | 18 ++++++++++++++---- Makefile | 20 ++++++++++++++++++-- programs/Makefile | 49 +++++++++++++++++++++++++++++++++---------------- 3 files changed, 65 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index 472ca18..c2d92e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,24 @@ language: c compiler: gcc -script: make test +script: make test-travis before_install: - - sudo apt-get update -qq + - sudo apt-get update -qq - sudo apt-get install -qq gcc-multilib - sudo apt-get install -qq valgrind env: - - LZ4_TRAVIS_CI_ENV=-m32 - - LZ4_TRAVIS_CI_ENV=-m64 + - LZ4_TRAVIS_CI_ENV=-dist + - LZ4_TRAVIS_CI_ENV=-examples + - LZ4_TRAVIS_CI_ENV=-lz4 + - LZ4_TRAVIS_CI_ENV=-lz4c + - LZ4_TRAVIS_CI_ENV=-lz4c32 + - LZ4_TRAVIS_CI_ENV=-fullbench + - LZ4_TRAVIS_CI_ENV=-fullbench32 + - LZ4_TRAVIS_CI_ENV=-fuzzer + - LZ4_TRAVIS_CI_ENV=-fuzzer32 + - LZ4_TRAVIS_CI_ENV=-frametest + - LZ4_TRAVIS_CI_ENV=-frametest32 + - LZ4_TRAVIS_CI_ENV=-mem matrix: fast_finish: true diff --git a/Makefile b/Makefile index a46cb41..225b21c 100644 --- a/Makefile +++ b/Makefile @@ -89,6 +89,16 @@ NONTEXT = images/image00.png images/image01.png images/image02.png \ SOURCES = $(TEXT) $(NONTEXT) +# Select test target for Travis CI's Build Matrix +ifeq ($(LZ4_TRAVIS_CI_ENV),-dist) +TRAVIS_TARGET=dist +else ifeq ($(LZ4_TRAVIS_CI_ENV),-examples) +TRAVIS_TARGET=examples +else +TRAVIS_TARGET=test-prg +endif + + default: liblz4 @cd $(PRGDIR); $(MAKE) -e @@ -167,8 +177,14 @@ dist: clean @echo Distribution $(DISTRIBNAME) built test: - make dist - @cd examples; $(MAKE) -e $@ @cd $(PRGDIR); $(MAKE) -e $@ +test-travis: $(TRAVIS_TARGET) + +examples: + cd examples; $(MAKE) -e test + +test-prg: + @cd $(PRGDIR); $(MAKE) -e test-travis + endif diff --git a/programs/Makefile b/programs/Makefile index 00af2f9..fa2f391 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -45,15 +45,6 @@ LZ4DIR=.. TEST_FILES = COPYING TEST_TARGETS=test-native -BENCH_NB=-i5 - -# Minimize test target for Travis CI's Build Matrix -ifeq ($(LZ4_TRAVIS_CI_ENV),-m32) -TEST_TARGETS=test-force32 -BENCH_NB=-i1 -else ifeq ($(LZ4_TRAVIS_CI_ENV),-m64) -BENCH_NB=-i1 -endif # Define *.exe as extension for Windows systems @@ -66,9 +57,35 @@ VOID = /dev/null endif +# Select test target for Travis CI's Build Matrix +ifeq ($(LZ4_TRAVIS_CI_ENV),-lz4) +TRAVIS_TARGET=test-lz4 +else ifeq ($(LZ4_TRAVIS_CI_ENV),-lz4c) +TRAVIS_TARGET=test-lz4c +else ifeq ($(LZ4_TRAVIS_CI_ENV),-lz4c32) +TRAVIS_TARGET=test-lz4c32 +else ifeq ($(LZ4_TRAVIS_CI_ENV),-fullbench) +TRAVIS_TARGET=test-fullbench +else ifeq ($(LZ4_TRAVIS_CI_ENV),-fullbench32) +TRAVIS_TARGET=test-fullbench32 +else ifeq ($(LZ4_TRAVIS_CI_ENV),-fuzzer) +TRAVIS_TARGET=test-fuzzer +else ifeq ($(LZ4_TRAVIS_CI_ENV),-fuzzer32) +TRAVIS_TARGET=test-fuzzer32 +else ifeq ($(LZ4_TRAVIS_CI_ENV),-frametest) +TRAVIS_TARGET=test-frametest +else ifeq ($(LZ4_TRAVIS_CI_ENV),-frametest32) +TRAVIS_TARGET=test-frametest32 +else ifeq ($(LZ4_TRAVIS_CI_ENV),-mem) +TRAVIS_TARGET=test-mem +else +TRAVIS_TARGET=test +endif + + default: lz4 lz4c -all: lz4 lz4c lz4c32 fullbench fullbench32 fuzzer fuzzer32 frametest datagen +all: lz4 lz4c lz4c32 fullbench fullbench32 fuzzer fuzzer32 frametest frametest32 datagen lz4: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c bench.c lz4io.c lz4cli.c $(CC) $(FLAGS) -DDISABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT) @@ -136,13 +153,13 @@ uninstall: [ -f $(DESTDIR)$(MANDIR)/lz4cat.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4cat.1 @echo lz4 successfully uninstalled -test-native: test-lz4 test-lz4c test-frame test-fullbench test-fuzzer test-mem +test: test-lz4 test-lz4c test-frame test-fullbench test-fuzzer test-mem -test-force32: test-lz4c32 test-frame32 test-fullbench32 test-fuzzer32 test-mem32 +test32: test-lz4c32 test-frame32 test-fullbench32 test-fuzzer32 test-mem32 -test-all: test-native test-force32 +test-all: test test32 -test: $(TEST_TARGETS) +test-travis: $(TRAVIS_TARGET) test-lz4: lz4 datagen ./datagen -g16KB | ./lz4 -9 | ./lz4 -vdq > $(VOID) @@ -162,10 +179,10 @@ test-lz4c32: lz4 lz4c32 lz4 datagen ./datagen -g6GB | ./lz4c32 -vqB5D | ./lz4c32 -vdq > $(VOID) test-fullbench: fullbench - ./fullbench --no-prompt $(BENCH_NB) $(TEST_FILES) + ./fullbench --no-prompt $(TEST_FILES) test-fullbench32: fullbench32 - ./fullbench32 --no-prompt $(BENCH_NB) $(TEST_FILES) + ./fullbench32 --no-prompt $(TEST_FILES) test-fuzzer: fuzzer ./fuzzer -- cgit v0.12 From d4f2d8061e0a69016d140720dfd96fb80dfe8445 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 28 Oct 2014 15:58:25 +0100 Subject: Fixed : frametest for Travis CI build matrix --- programs/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index fa2f391..3699a64 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -153,9 +153,9 @@ uninstall: [ -f $(DESTDIR)$(MANDIR)/lz4cat.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4cat.1 @echo lz4 successfully uninstalled -test: test-lz4 test-lz4c test-frame test-fullbench test-fuzzer test-mem +test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-mem -test32: test-lz4c32 test-frame32 test-fullbench32 test-fuzzer32 test-mem32 +test32: test-lz4c32 test-frametest32 test-fullbench32 test-fuzzer32 test-mem32 test-all: test test32 @@ -190,10 +190,10 @@ test-fuzzer: fuzzer test-fuzzer32: fuzzer32 ./fuzzer32 -test-frame: frametest +test-frametest: frametest ./frametest -test-frame32: frametest32 +test-frametest32: frametest32 ./frametest32 test-mem: lz4 datagen frametest -- cgit v0.12 From 5a5a15969ea853d461b4f7f631c10b999de43c9c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 29 Oct 2014 01:39:47 +0100 Subject: minor fuzzer test correction --- programs/fuzzer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/programs/fuzzer.c b/programs/fuzzer.c index b7721c8..63c3ab9 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -600,6 +600,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do FUZ_DISPLAYTEST; dict -= (FUZ_rand(&randState) & 7); // even bigger separation if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; + LZ4_resetStreamHC (&LZ4dictHC, FUZ_rand(&randState) & 0x7); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); blockContinueCompressedSize = LZ4_compressHC_continue(&LZ4dictHC, block, compressedBuffer, blockSize); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compressHC_continue failed"); -- cgit v0.12 From 55e8a47474fb63d10a0e6c08d3f761795a9a724f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 29 Oct 2014 01:47:43 +0100 Subject: Added : valgrind test for fuzzer --- programs/Makefile | 3 ++- programs/fuzzer.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index 3699a64..ad64340 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -196,7 +196,7 @@ test-frametest: frametest test-frametest32: frametest32 ./frametest32 -test-mem: lz4 datagen frametest +test-mem: lz4 datagen fuzzer frametest ./datagen -g16KB > tmp valgrind --leak-check=yes ./lz4 -9 -BD -f tmp /dev/null ./datagen -g16MB > tmp @@ -204,6 +204,7 @@ test-mem: lz4 datagen frametest ./datagen -g256MB > tmp valgrind --leak-check=yes ./lz4 -B4D -f tmp /dev/null rm tmp + valgrind --leak-check=yes ./fuzzer -i50 -t0 valgrind --leak-check=yes ./frametest -i100 test-mem32: lz4c32 datagen diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 63c3ab9..3a2ec34 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -1058,7 +1058,7 @@ int main(int argc, char** argv) case 't': /* select starting test nb */ argument++; - testNb=0; + testNb=0; seedset=1; while ((*argument>='0') && (*argument<='9')) { testNb *= 10; -- cgit v0.12 From d9927ea693262df817a43bca5b4ea6d299f7db5f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 1 Nov 2014 21:58:31 +0100 Subject: Updated xxHash to r38 --- xxhash.c | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/xxhash.c b/xxhash.c index 529d69c..e6c2f31 100644 --- a/xxhash.c +++ b/xxhash.c @@ -84,14 +84,8 @@ You can contact the author at : // Modify the local functions below should you wish to use some other memory routines // for malloc(), free() #include -FORCE_INLINE void* XXH_malloc(size_t s) -{ - return malloc(s); -} -FORCE_INLINE void XXH_free (void* p) -{ - free(p); -} +FORCE_INLINE void* XXH_malloc(size_t s) { return malloc(s); } +FORCE_INLINE void XXH_free (void* p) { free(p); } // for memcpy() #include FORCE_INLINE void* XXH_memcpy(void* dest, const void* src, size_t size) @@ -535,22 +529,22 @@ typedef struct XXH32_state_t* XXH32_createState(void) { XXH_STATIC_ASSERT(sizeof(XXH32_state_t) >= sizeof(XXH_istate32_t)); // A compilation error here means XXH32_state_t is not large enough - return (XXH32_state_t*)malloc(sizeof(XXH32_state_t)); + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); } XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) { - free(statePtr); + XXH_free(statePtr); return XXH_OK; }; XXH64_state_t* XXH64_createState(void) { XXH_STATIC_ASSERT(sizeof(XXH64_state_t) >= sizeof(XXH_istate64_t)); // A compilation error here means XXH64_state_t is not large enough - return (XXH64_state_t*)malloc(sizeof(XXH64_state_t)); + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); } XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) { - free(statePtr); + XXH_free(statePtr); return XXH_OK; }; -- cgit v0.12 From 61289dea1d94510a4d6386bc44c08d3d15083123 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 2 Nov 2014 22:32:12 +0100 Subject: Optimized LZ4_saveDictHC() --- lz4.c | 6 ++--- lz4.h | 2 +- lz4hc.c | 51 ++++++++++++++++++++++++++--------------- lz4hc.h | 2 +- programs/fullbench.c | 65 +++++++++++++++++++++++++++++++++------------------- 5 files changed, 79 insertions(+), 47 deletions(-) diff --git a/lz4.c b/lz4.c index e8693ae..9345e74 100644 --- a/lz4.c +++ b/lz4.c @@ -701,6 +701,7 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream) LZ4_stream_t* LZ4_createStream(void) { LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(4, LZ4_STREAMSIZE_U32); + LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ LZ4_resetStream(lz4s); return lz4s; } @@ -719,14 +720,13 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) const BYTE* const dictEnd = p + dictSize; const BYTE* base; - LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ if (dict->initCheck) LZ4_resetStream(LZ4_dict); /* Uninitialized structure detected */ if (dictSize < MINMATCH) { dict->dictionary = NULL; dict->dictSize = 0; - return 1; + return 0; } if (p <= dictEnd - 64 KB) p = dictEnd - 64 KB; @@ -741,7 +741,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) p+=3; } - return 1; + return dict->dictSize; } diff --git a/lz4.h b/lz4.h index 0a8a772..3c35c6e 100644 --- a/lz4.h +++ b/lz4.h @@ -200,7 +200,7 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_streamPtr); * Use this function to load a static dictionary into LZ4_stream. * Any previous data will be forgotten, only 'dictionary' will remain in memory. * Loading a size of 0 is allowed. - * Return : 1 if OK, 0 if error + * Return : dictionary size, in bytes (necessarily <= 64 KB) */ int LZ4_loadDict (LZ4_stream_t* LZ4_streamPtr, const char* dictionary, int dictSize); diff --git a/lz4hc.c b/lz4hc.c index 1422a7c..8f6f25b 100644 --- a/lz4hc.c +++ b/lz4hc.c @@ -250,14 +250,14 @@ typedef struct { U32 hashTable[HASHTABLESIZE]; U16 chainTable[MAXD]; - const BYTE* inputBuffer; - const BYTE* base; - const BYTE* end; - const BYTE* dictBase; - U32 dictLimit; + const BYTE* end; /* next block here to keep current prefix as prefix */ + const BYTE* base; /* All index relative to this position */ + const BYTE* dictBase; /* alternate base for extDict */ + U32 dictLimit; /* below that point, need extDict */ + U32 lowLimit; /* below that point, no more dict */ U32 nextToUpdate; U32 compressionLevel; - U32 lowLimit; + const BYTE* inputBuffer; /* deprecated */ } LZ4HC_Data_Structure; @@ -342,7 +342,7 @@ FORCE_INLINE int LZ4_NbCommonBytes (register U32 val) #endif -FORCE_INLINE void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* base) +static void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* base) { MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); @@ -857,10 +857,16 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) { - LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) dictionary); - if (dictSize >= 4) LZ4HC_Insert ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*)dictionary +(dictSize-3)); - ((LZ4HC_Data_Structure*) LZ4_streamHCPtr)->end = (const BYTE*)dictionary + dictSize; - return 1; + LZ4HC_Data_Structure* streamPtr = (LZ4HC_Data_Structure*) LZ4_streamHCPtr; + if (dictSize > 64 KB) + { + dictionary += dictSize - 64 KB; + dictSize = 64 KB; + } + LZ4HC_init (streamPtr, (const BYTE*)dictionary); + if (dictSize >= 4) LZ4HC_Insert (streamPtr, (const BYTE*)dictionary +(dictSize-3)); + streamPtr->end = (const BYTE*)dictionary + dictSize; + return dictSize; } @@ -917,20 +923,27 @@ int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, cons int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize) { - LZ4HC_Data_Structure* sp = (LZ4HC_Data_Structure*)LZ4_streamHCPtr; + LZ4HC_Data_Structure* streamPtr = (LZ4HC_Data_Structure*)LZ4_streamHCPtr; + int prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit)); if (dictSize > 64 KB) dictSize = 64 KB; - if (dictSize < 0) dictSize = 0; - if (dictSize > (sp->end - (sp->base + sp->lowLimit))) dictSize = (int)(sp->end - (sp->base + sp->lowLimit)); - memcpy(safeBuffer, sp->end - dictSize, dictSize); - LZ4_loadDictHC(LZ4_streamHCPtr, safeBuffer, dictSize); + if (dictSize < 4) dictSize = 0; + if (dictSize > prefixSize) dictSize = prefixSize; + memcpy(safeBuffer, streamPtr->end - dictSize, dictSize); + //LZ4_loadDictHC(LZ4_streamHCPtr, safeBuffer, dictSize); + { + U32 endIndex = (U32)(streamPtr->end - streamPtr->base); + streamPtr->end = (const BYTE*)safeBuffer + dictSize; + streamPtr->base = streamPtr->end - endIndex; + streamPtr->dictLimit = endIndex - dictSize; + streamPtr->lowLimit = endIndex - dictSize; + } return dictSize; } - /*********************************** -Deprecated Streaming functions -***********************************/ + * Deprecated Functions + ***********************************/ int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; } int LZ4_resetStreamStateHC(void* state, const char* inputBuffer) diff --git a/lz4hc.h b/lz4hc.h index 5fbf398..23ad7c1 100644 --- a/lz4hc.h +++ b/lz4hc.h @@ -145,7 +145,7 @@ They work like usual LZ4_compressHC() or LZ4_compressHC_limitedOutput(), but use Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression. If, for any reason, previous data block can't be preserved in memory during next compression block, -you can save it to a safer memory space, +you must save it to a safer memory space, using LZ4_saveDictHC(). */ diff --git a/programs/fullbench.c b/programs/fullbench.c index 4caea16..3592223 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -326,6 +326,20 @@ static int local_LZ4F_compressFrame(const char* in, char* out, int inSize) return (int)LZ4F_compressFrame(out, 2*inSize + 16, in, inSize, NULL); } +static int local_LZ4_saveDict(const char* in, char* out, int inSize) +{ + (void)in; + return LZ4_saveDict(&LZ4_dict, out, inSize); +} + +LZ4_streamHC_t LZ4_dictHC; +static int local_LZ4_saveDictHC(const char* in, char* out, int inSize) +{ + (void)in; + return LZ4_saveDictHC(&LZ4_dictHC, out, inSize); +} + + static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize) { (void)inSize; @@ -386,7 +400,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) { int fileIdx=0; char* orig_buff; -# define NB_COMPRESSION_ALGORITHMS 14 +# define NB_COMPRESSION_ALGORITHMS 16 double totalCTime[NB_COMPRESSION_ALGORITHMS+1] = {0}; double totalCSize[NB_COMPRESSION_ALGORITHMS+1] = {0}; # define NB_DECOMPRESSION_ALGORITHMS 9 @@ -416,8 +430,8 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) // Init - stateLZ4 = malloc(LZ4_sizeofState()); - stateLZ4HC = malloc(LZ4_sizeofStateHC()); + stateLZ4 = LZ4_createStream(); + stateLZ4HC = LZ4_createStreamHC(); // Check file existence inFileName = fileNamesTable[fileIdx++]; @@ -483,23 +497,6 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) DISPLAY("\r%79s\r", ""); DISPLAY(" %s : \n", inFileName); - // Init chunks data - { - int i; - size_t remaining = benchedSize; - char* in = orig_buff; - char* out = compressed_buff; - nbChunks = (int) ((int)benchedSize / chunkSize) + 1; - for (i=0; i chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } - chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; - chunkP[i].compressedSize = 0; - } - } - // Compression Algorithms for (cAlgNb=1; (cAlgNb <= NB_COMPRESSION_ALGORITHMS) && (compressionTest); cAlgNb++) { @@ -508,6 +505,23 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) void* (*initFunction)(const char*) = NULL; double bestTime = 100000000.; + // Init data chunks + { + int i; + size_t remaining = benchedSize; + char* in = orig_buff; + char* out = compressed_buff; + nbChunks = (int) ((int)benchedSize / chunkSize) + 1; + for (i=0; i chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } + chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; + chunkP[i].compressedSize = 0; + } + } + if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != cAlgNb)) continue; switch(cAlgNb) @@ -528,6 +542,12 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) case 14: compressionFunction = local_LZ4F_compressFrame; compressorName = "LZ4F_compressFrame"; chunkP[0].origSize = (int)benchedSize; nbChunks=1; break; + case 15: compressionFunction = local_LZ4_saveDict; compressorName = "LZ4_saveDict"; + LZ4_loadDict(&LZ4_dict, chunkP[0].origBuffer, chunkP[0].origSize); + break; + case 16: compressionFunction = local_LZ4_saveDictHC; compressorName = "LZ4_saveDictHC"; + LZ4_loadDictHC(&LZ4_dictHC, chunkP[0].origBuffer, chunkP[0].origSize); + break; default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1; } @@ -537,7 +557,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) int milliTime; PROGRESS("%1i-%-26.26s : %9i ->\r", loopNb, compressorName, (int)benchedSize); - { size_t i; for (i=0; i Date: Mon, 3 Nov 2014 09:18:15 +0100 Subject: fullbench : display Algorithm Nb --- programs/fullbench.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/programs/fullbench.c b/programs/fullbench.c index 3592223..cdf1d1d 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -556,7 +556,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) double averageTime; int milliTime; - PROGRESS("%1i-%-26.26s : %9i ->\r", loopNb, compressorName, (int)benchedSize); + PROGRESS("%1i- %-28.28s :%9i ->\r", loopNb, compressorName, (int)benchedSize); { size_t i; for (i=0; i %9i (%5.2f%%),%7.1f MB/s\r", loopNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); + PROGRESS("%1i- %-28.28s :%9i ->%9i (%5.2f%%),%7.1f MB/s\r", loopNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); } if (ratio<100.) - DISPLAY("%-28.28s : %9i -> %9i (%5.2f%%),%7.1f MB/s\n", compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); + DISPLAY("%2i-%-28.28s :%9i ->%9i (%5.2f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); else - DISPLAY("%-28.28s : %9i -> %9i (%5.1f%%),%7.1f MB/s\n", compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); + DISPLAY("%2i-%-28.28s :%9i ->%9i (%5.1f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); totalCTime[cAlgNb] += bestTime; totalCSize[cAlgNb] += cSize; @@ -637,7 +637,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) int milliTime; U32 crcDecoded; - PROGRESS("%1i-%-29.29s :%10i ->\r", loopNb, dName, (int)benchedSize); + PROGRESS("%1i- %-29.29s :%10i ->\r", loopNb, dName, (int)benchedSize); nb_loops = 0; milliTime = BMK_GetMilliStart(); @@ -657,14 +657,14 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) averageTime = (double)milliTime / nb_loops; if (averageTime < bestTime) bestTime = averageTime; - PROGRESS("%1i-%-29.29s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.); + PROGRESS("%1i- %-29.29s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.); // CRC Checking crcDecoded = XXH32(orig_buff, (int)benchedSize, 0); if (crcOriginal!=crcDecoded) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOriginal, (unsigned)crcDecoded); exit(1); } } - DISPLAY("%-31.31s :%10i -> %7.1f MB/s\n", dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.); + DISPLAY("%2i-%-29.29s :%10i -> %7.1f MB/s\n", dAlgNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.); totalDTime[dAlgNb] += bestTime; } -- cgit v0.12 From 3de7f00a0b83f0f8ad861789d8da1ad697b1f05a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 3 Nov 2014 09:29:45 +0100 Subject: Updated ring buffer examples --- examples/HCStreaming_ringBuffer.c | 25 ++++++++++++------------- examples/blockStreaming_ringBuffer.c | 22 ++++++++++++++++++---- 2 files changed, 30 insertions(+), 17 deletions(-) mode change 100755 => 100644 examples/HCStreaming_ringBuffer.c diff --git a/examples/HCStreaming_ringBuffer.c b/examples/HCStreaming_ringBuffer.c old mode 100755 new mode 100644 index 53e14ca..afe77ae --- a/examples/HCStreaming_ringBuffer.c +++ b/examples/HCStreaming_ringBuffer.c @@ -3,8 +3,8 @@ /************************************** -Compiler Options -**************************************/ + * Compiler Options + **************************************/ #ifdef _MSC_VER /* Visual Studio */ # define _CRT_SECURE_NO_WARNINGS // for MSVC # define snprintf sprintf_s @@ -16,10 +16,9 @@ Compiler Options #endif - /************************************** -Includes -**************************************/ + * Includes + **************************************/ #include "lz4hc.h" #include "lz4.h" @@ -98,7 +97,7 @@ void test_decompress(FILE* outFp, FILE* inpFp) LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body; unsigned done = 0; - for(;;) + for(;;) { int cmpBytes = 0; char cmpBuf[LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)]; @@ -106,11 +105,11 @@ void test_decompress(FILE* outFp, FILE* inpFp) { const size_t r0 = read_int32(inpFp, &cmpBytes); size_t r1; - if(r0 != 1 || cmpBytes <= 0) + if(r0 != 1 || cmpBytes <= 0) break; r1 = read_bin(inpFp, cmpBuf, cmpBytes); - if(r1 != (size_t) cmpBytes) + if(r1 != (size_t) cmpBytes) break; } @@ -118,7 +117,7 @@ void test_decompress(FILE* outFp, FILE* inpFp) char* const decPtr = &decBuf[decOffset]; const int decBytes = LZ4_decompress_safe_continue( lz4StreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES); - if(decBytes <= 0) + if(decBytes <= 0) break; done += decBytes; @@ -126,7 +125,7 @@ void test_decompress(FILE* outFp, FILE* inpFp) write_bin(outFp, decPtr, decBytes); // Wraparound the ringbuffer offset - if(decOffset >= DEC_BUFFER_BYTES - MESSAGE_MAX_BYTES) + if(decOffset >= DEC_BUFFER_BYTES - MESSAGE_MAX_BYTES) decOffset = 0; } } @@ -192,9 +191,9 @@ int main(int argc, char** argv) snprintf(lz4Filename, 256, "%s.lz4s-%d", argv[fileID], 9); snprintf(decFilename, 256, "%s.lz4s-%d.dec", argv[fileID], 9); - printf("inp = [%s]\n", inpFilename); - printf("lz4 = [%s]\n", lz4Filename); - printf("dec = [%s]\n", decFilename); + printf("input = [%s]\n", inpFilename); + printf("lz4 = [%s]\n", lz4Filename); + printf("decoded = [%s]\n", decFilename); // compress { diff --git a/examples/blockStreaming_ringBuffer.c b/examples/blockStreaming_ringBuffer.c index bfdbed1..eda721f 100644 --- a/examples/blockStreaming_ringBuffer.c +++ b/examples/blockStreaming_ringBuffer.c @@ -1,18 +1,32 @@ // LZ4 streaming API example : ring buffer -// Copyright : Takayuki Matsuoka +// Based on sample code from Takayuki Matsuoka -#define _CRT_SECURE_NO_WARNINGS // for MSVC -#include "lz4.h" +/************************************** + * Compiler Options + **************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define _CRT_SECURE_NO_WARNINGS // for MSVC +# define snprintf sprintf_s +#endif +#ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ +#endif + +/************************************** + * Includes + **************************************/ #include #include #include #include +#include "lz4.h" + enum { MESSAGE_MAX_BYTES = 1024, - RING_BUFFER_BYTES = 1024 * 256 + MESSAGE_MAX_BYTES, + RING_BUFFER_BYTES = 1024 * 8 + MESSAGE_MAX_BYTES, DICT_BYTES = 65536, }; -- cgit v0.12 From 7050d4b7672f164144f0cef0a95aa68ad71cf9e6 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 4 Nov 2014 10:04:37 +0100 Subject: Updated unsynchronized buffers tests --- examples/HCStreaming_ringBuffer.c | 2 +- examples/blockStreaming_ringBuffer.c | 6 ++-- programs/fuzzer.c | 55 ++++++++++++++++++++++++++++++++---- 3 files changed, 54 insertions(+), 9 deletions(-) diff --git a/examples/HCStreaming_ringBuffer.c b/examples/HCStreaming_ringBuffer.c index afe77ae..dfa2670 100644 --- a/examples/HCStreaming_ringBuffer.c +++ b/examples/HCStreaming_ringBuffer.c @@ -30,7 +30,7 @@ enum { MESSAGE_MAX_BYTES = 1024, RING_BUFFER_BYTES = 1024 * 8 + MESSAGE_MAX_BYTES, - DEC_BUFFER_BYTES = RING_BUFFER_BYTES + MESSAGE_MAX_BYTES + DEC_BUFFER_BYTES = RING_BUFFER_BYTES + MESSAGE_MAX_BYTES // Intentionally larger to test unsynchronized ring buffers }; diff --git a/examples/blockStreaming_ringBuffer.c b/examples/blockStreaming_ringBuffer.c index eda721f..beb5e66 100644 --- a/examples/blockStreaming_ringBuffer.c +++ b/examples/blockStreaming_ringBuffer.c @@ -27,7 +27,7 @@ enum { MESSAGE_MAX_BYTES = 1024, RING_BUFFER_BYTES = 1024 * 8 + MESSAGE_MAX_BYTES, - DICT_BYTES = 65536, + DECODE_RING_BUFFER = RING_BUFFER_BYTES + MESSAGE_MAX_BYTES // Intentionally larger, to test unsynchronized ring buffers }; @@ -83,7 +83,7 @@ void test_compress(FILE* outFp, FILE* inpFp) void test_decompress(FILE* outFp, FILE* inpFp) { - static char decBuf[RING_BUFFER_BYTES]; + static char decBuf[DECODE_RING_BUFFER]; int decOffset = 0; LZ4_streamDecode_t lz4StreamDecode_body = { 0 }; LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body; @@ -109,7 +109,7 @@ void test_decompress(FILE* outFp, FILE* inpFp) write_bin(outFp, decPtr, decBytes); // Wraparound the ringbuffer offset - if(decOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES) decOffset = 0; + if(decOffset >= DECODE_RING_BUFFER - MESSAGE_MAX_BYTES) decOffset = 0; } } } diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 3a2ec34..17f7c43 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -663,8 +663,8 @@ _output_error: } -#define testInputSize (128 KB) -#define testCompressedSize (64 KB) +#define testInputSize (192 KB) +#define testCompressedSize (128 KB) #define ringBufferSize (8 KB) static void FUZ_unitTests(void) @@ -786,15 +786,15 @@ static void FUZ_unitTests(void) FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 64 KB); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() dictionary decompression failed"); + FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed"); crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() simple dictionary decompression test : corruption"); // multiple HC compression test with dictionary { int result1, result2; int segSize = testCompressedSize / 2; - crcOrig = XXH64(testInput + segSize, 64 KB, 0); + crcOrig = XXH64(testInput + segSize, testCompressedSize, 0); LZ4_resetStreamHC(&sHC, 0); LZ4_loadDictHC(&sHC, testInput, segSize); result1 = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1); @@ -915,6 +915,48 @@ static void FUZ_unitTests(void) } } + // small decoder-side ring buffer test + { + XXH64_state_t xxhOrig; + XXH64_state_t xxhNew; + LZ4_streamDecode_t decodeState; + const U32 maxMessageSizeLog = 10; + const U32 maxMessageSizeMask = (1< dBufferSize) dNext = 0; + } + } + // long stream test ; Warning : very long test ! if (1) { @@ -939,6 +981,9 @@ static void FUZ_unitTests(void) FUZ_displayUpdate((U32)(totalTestDone >> 20)); + if (testStart == oldStart + oldSize) // Corner case not covered by this test (LZ4_decompress_safe_usingDict() limitation) + testStart++; + XXH64_update(&crcOrigState, testInput + testStart, testSize); crcOrig = XXH64_digest(&crcOrigState); -- cgit v0.12 From d01bb4a193338e851e76bd9b80f79bb9bdbc7b31 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 4 Nov 2014 10:32:50 +0100 Subject: Makefile : added -Wcast-align compilation warning --- Makefile | 2 +- programs/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 225b21c..8cf6f8a 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ DESTDIR?= PREFIX ?= /usr CC := $(CC) CFLAGS ?= -O3 -CFLAGS += -I. -std=c99 -Wall -Wextra -Wundef -Wshadow -Wstrict-prototypes -DLZ4_VERSION=\"$(RELEASE)\" +CFLAGS += -I. -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -DLZ4_VERSION=\"$(RELEASE)\" LIBDIR?= $(PREFIX)/lib INCLUDEDIR=$(PREFIX)/include diff --git a/programs/Makefile b/programs/Makefile index ad64340..1a81d73 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -36,7 +36,7 @@ DESTDIR?= PREFIX ?= /usr CC := $(CC) CFLAGS ?= -O3 -CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wstrict-prototypes -DLZ4_VERSION=\"$(RELEASE)\" +CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -DLZ4_VERSION=\"$(RELEASE)\" FLAGS = -I.. $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) BINDIR=$(PREFIX)/bin -- cgit v0.12 From 4dbd2faebb345e4b9d888135e9adffe500bdbb38 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 4 Nov 2014 10:50:17 +0100 Subject: Added : cmake test cmake : corrected xxhash path (thanks to j.magnuson) --- .travis.yml | 1 + Makefile | 5 +++++ cmake_unofficial/CMakeLists.txt | 9 +++++---- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index c2d92e4..2a2623c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ before_install: env: - LZ4_TRAVIS_CI_ENV=-dist + - LZ4_TRAVIS_CI_ENV=-cmake - LZ4_TRAVIS_CI_ENV=-examples - LZ4_TRAVIS_CI_ENV=-lz4 - LZ4_TRAVIS_CI_ENV=-lz4c diff --git a/Makefile b/Makefile index 8cf6f8a..c0b7a6f 100644 --- a/Makefile +++ b/Makefile @@ -92,6 +92,8 @@ SOURCES = $(TEXT) $(NONTEXT) # Select test target for Travis CI's Build Matrix ifeq ($(LZ4_TRAVIS_CI_ENV),-dist) TRAVIS_TARGET=dist +else ifeq ($(LZ4_TRAVIS_CI_ENV),-cmake) +TRAVIS_TARGET=cmake else ifeq ($(LZ4_TRAVIS_CI_ENV),-examples) TRAVIS_TARGET=examples else @@ -181,6 +183,9 @@ test: test-travis: $(TRAVIS_TARGET) +cmake: + @cd cmake_unofficial; cmake CMakeLists.txt; $(MAKE) + examples: cd examples; $(MAKE) -e test diff --git a/cmake_unofficial/CMakeLists.txt b/cmake_unofficial/CMakeLists.txt index fab1946..1d70590 100644 --- a/cmake_unofficial/CMakeLists.txt +++ b/cmake_unofficial/CMakeLists.txt @@ -1,9 +1,8 @@ PROJECT(LZ4 C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "LZ4 compression library") set(CPACK_PACKAGE_VERSION_MAJOR 1) -set(CPACK_PACKAGE_VERSION_MINOR 3) -set(CPACK_PACKAGE_VERSION_PATCH r122) -#set(CPACK_RESOURCE_FILE_LICENSE ${CMAKE_CURRENT_BINARY_DIR}/COPYING_LGPL) +set(CPACK_PACKAGE_VERSION_MINOR 4) +set(CPACK_PACKAGE_VERSION_PATCH r124) set(VERSION_STRING " \"${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}\" ") include(CPack) @@ -27,7 +26,7 @@ endif() set(LZ4_DIR ../) set(PRG_DIR ../programs/) set(LZ4_SRCS_LIB ${LZ4_DIR}lz4.c ${LZ4_DIR}lz4hc.c ${LZ4_DIR}lz4.h ${LZ4_DIR}lz4hc.h) -set(LZ4_SRCS ${PRG_DIR}xxhash.c ${PRG_DIR}bench.c ${PRG_DIR}lz4cli.c ${PRG_DIR}lz4io.c) +set(LZ4_SRCS ${LZ4_DIR}xxhash.c ${PRG_DIR}bench.c ${PRG_DIR}lz4cli.c ${PRG_DIR}lz4io.c) if(BUILD_TOOLS AND NOT BUILD_LIBS) set(LZ4_SRCS ${LZ4_SRCS} ${LZ4_SRCS_LIB}) @@ -72,7 +71,9 @@ endif() ADD_DEFINITIONS("-Wall") ADD_DEFINITIONS("-Wextra") ADD_DEFINITIONS("-Wundef") +ADD_DEFINITIONS("-Wshadow") ADD_DEFINITIONS("-Wcast-align") +ADD_DEFINITIONS("-Wstrict-prototypes") ADD_DEFINITIONS("-std=c99") ADD_DEFINITIONS("-DLZ4_VERSION=\"${CPACK_PACKAGE_VERSION_PATCH}\"") INCLUDE_DIRECTORIES (${LZ4_DIR}) -- cgit v0.12 From d7b9e31702137c589d070776427ff57d4740eb8d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 4 Nov 2014 11:07:04 +0100 Subject: Fixed issue #31 : lz4.c warning under visual --- lz4.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lz4.c b/lz4.c index 9345e74..198b581 100644 --- a/lz4.c +++ b/lz4.c @@ -289,7 +289,7 @@ typedef enum { full = 0, partial = 1 } earlyEnd_directive; /************************************** Macros **************************************/ -#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(!!(c)) }; } /* use only *after* variable declarations */ +#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ #if LZ4_ARCH64 || !defined(__GNUC__) # define LZ4_WILDCOPY(d,s,e) { do { LZ4_COPY8(d,s) } while (d=e; */ #else -- cgit v0.12 From 89d8b98d0dd896dccf6964fa3c6bb6ae6a2349b0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 4 Nov 2014 12:11:14 +0100 Subject: Fixed issue 44 : stopped concatenated stream on null size frame (thanks to Brendan O'Connor) --- programs/Makefile | 12 +++++++++++- programs/lz4io.c | 25 ++++++++++--------------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index 1a81d73..ba08cfb 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -119,7 +119,7 @@ datagen : datagen.c clean: - @rm -f core *.o \ + @rm -f core *.o *.test \ lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) \ fullbench$(EXT) fullbench32$(EXT) \ fuzzer$(EXT) fuzzer32$(EXT) \ @@ -166,6 +166,16 @@ test-lz4: lz4 datagen ./datagen | ./lz4 | ./lz4 -vdq > $(VOID) ./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -vdq > $(VOID) ./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -vdq > $(VOID) + echo -n > empty.test + echo hi > nonempty.test + cat nonempty.test empty.test nonempty.test > orig.test + @./lz4 -zq empty.test > empty.lz4.test + @./lz4 -zq nonempty.test > nonempty.lz4.test + cat nonempty.lz4.test empty.lz4.test nonempty.lz4.test > concat.lz4.test + ./lz4 -d concat.lz4.test > result.test + sdiff orig.test result.test + @rm *.test + test-lz4c: lz4c datagen diff --git a/programs/lz4io.c b/programs/lz4io.c index 6de5206..3a84866 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -33,16 +33,9 @@ // Compiler Options //************************************** #ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline # define _CRT_SECURE_NO_WARNINGS # define _CRT_SECURE_NO_DEPRECATE // VS2005 # pragma warning(disable : 4127) // disable: C4127: conditional expression is constant -#else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif #endif #define _LARGE_FILES // Large file support on 32-bits AIX @@ -91,7 +84,7 @@ #elif GCC_VERSION >= 403 # define swap32 __builtin_bswap32 #else - static inline unsigned int swap32(unsigned int x) + static unsigned int swap32(unsigned int x) { return ((x << 24) & 0xff000000 ) | ((x << 8) & 0x00ff0000 ) | @@ -289,8 +282,8 @@ static int get_fileHandle(char* input_filename, char* output_filename, FILE** pf } -// LZ4IO_compressFilename_Legacy : This function is "hidden" (not published in .h) -// Its purpose is to generate compressed streams using the old 'legacy' format +// LZ4IO_compressFilename_Legacy : This function is intentionally "hidden" (not published in .h) +// It generates compressed streams using the old 'legacy' format int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, int compressionlevel) { int (*compressionFunction)(const char*, char*, int); @@ -549,7 +542,7 @@ static int compress_file_blockDependency(char* input_filename, char* output_file } -FORCE_INLINE int LZ4_compress_limitedOutput_local(const char* src, char* dst, int size, int maxOut, int clevel) +static int LZ4_compress_limitedOutput_local(const char* src, char* dst, int size, int maxOut, int clevel) { (void)clevel; return LZ4_compress_limitedOutput(src, dst, size, maxOut); } int LZ4IO_compressFilename(char* input_filename, char* output_filename, int compressionLevel) @@ -888,6 +881,7 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) } +#define ENDOFSTREAM ((unsigned long long)-1) static unsigned long long selectDecoder( FILE* finput, FILE* foutput) { unsigned int magicNumber, size; @@ -896,7 +890,7 @@ static unsigned long long selectDecoder( FILE* finput, FILE* foutput) // Check Archive Header nbReadBytes = fread(&magicNumber, 1, MAGICNUMBER_SIZE, finput); - if (nbReadBytes==0) return 0; // EOF + if (nbReadBytes==0) return ENDOFSTREAM; // EOF if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(41, "Unrecognized header : Magic Number unreadable"); magicNumber = LITTLE_ENDIAN_32(magicNumber); // Convert to Little Endian format if (LZ4S_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4S_SKIPPABLE0; // fold skippable magic numbers @@ -920,7 +914,7 @@ static unsigned long long selectDecoder( FILE* finput, FILE* foutput) default: if (ftell(finput) == MAGICNUMBER_SIZE) EXM_THROW(44,"Unrecognized header : file cannot be decoded"); // Wrong magic number at the beginning of 1st stream DISPLAYLEVEL(2, "Stream followed by unrecognized data\n"); - return 0; + return ENDOFSTREAM; } } @@ -941,8 +935,9 @@ int LZ4IO_decompressFilename(char* input_filename, char* output_filename) do { decodedSize = selectDecoder(finput, foutput); - filesize += decodedSize; - } while (decodedSize); + if (decodedSize != ENDOFSTREAM) + filesize += decodedSize; + } while (decodedSize != ENDOFSTREAM); // Final Status end = clock(); -- cgit v0.12 From 4befab0fcc741101c4cfd4b81a6ca60ba5a4dafe Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 4 Nov 2014 20:04:43 +0100 Subject: lz4.h : updated library version number to 1.4.0 --- lz4.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lz4.h b/lz4.h index 3c35c6e..0350f6a 100644 --- a/lz4.h +++ b/lz4.h @@ -47,8 +47,8 @@ extern "C" { Version **************************************/ #define LZ4_VERSION_MAJOR 1 /* for major interface/format changes */ -#define LZ4_VERSION_MINOR 3 /* for minor interface/format changes */ -#define LZ4_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */ +#define LZ4_VERSION_MINOR 4 /* for minor interface/format changes */ +#define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) int LZ4_versionNumber (void); -- cgit v0.12 From 73554386ed8c9b08940411c0e62bbdca3216f8cd Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 4 Nov 2014 20:56:38 +0100 Subject: Makefile : refactored Travis CI tests --- .travis.yml | 27 ++++++++++++++------------- Makefile | 16 ++++++---------- programs/Makefile | 27 ++++----------------------- 3 files changed, 24 insertions(+), 46 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2a2623c..67bde9b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,19 +7,20 @@ before_install: - sudo apt-get install -qq valgrind env: - - LZ4_TRAVIS_CI_ENV=-dist - - LZ4_TRAVIS_CI_ENV=-cmake - - LZ4_TRAVIS_CI_ENV=-examples - - LZ4_TRAVIS_CI_ENV=-lz4 - - LZ4_TRAVIS_CI_ENV=-lz4c - - LZ4_TRAVIS_CI_ENV=-lz4c32 - - LZ4_TRAVIS_CI_ENV=-fullbench - - LZ4_TRAVIS_CI_ENV=-fullbench32 - - LZ4_TRAVIS_CI_ENV=-fuzzer - - LZ4_TRAVIS_CI_ENV=-fuzzer32 - - LZ4_TRAVIS_CI_ENV=-frametest - - LZ4_TRAVIS_CI_ENV=-frametest32 - - LZ4_TRAVIS_CI_ENV=-mem + - LZ4_TRAVIS_CI_ENV=liblz4 + - LZ4_TRAVIS_CI_ENV=streaming-examples + - LZ4_TRAVIS_CI_ENV=cmake + - LZ4_TRAVIS_CI_ENV=dist + - LZ4_TRAVIS_CI_ENV=test-lz4 + - LZ4_TRAVIS_CI_ENV=test-lz4c + - LZ4_TRAVIS_CI_ENV=test-lz4c32 + - LZ4_TRAVIS_CI_ENV=test-fullbench + - LZ4_TRAVIS_CI_ENV=test-fullbench32 + - LZ4_TRAVIS_CI_ENV=test-fuzzer + - LZ4_TRAVIS_CI_ENV=test-fuzzer32 + - LZ4_TRAVIS_CI_ENV=test-frametest + - LZ4_TRAVIS_CI_ENV=test-frametest32 + - LZ4_TRAVIS_CI_ENV=test-mem matrix: fast_finish: true diff --git a/Makefile b/Makefile index c0b7a6f..586ac5e 100644 --- a/Makefile +++ b/Makefile @@ -90,14 +90,10 @@ SOURCES = $(TEXT) $(NONTEXT) # Select test target for Travis CI's Build Matrix -ifeq ($(LZ4_TRAVIS_CI_ENV),-dist) -TRAVIS_TARGET=dist -else ifeq ($(LZ4_TRAVIS_CI_ENV),-cmake) -TRAVIS_TARGET=cmake -else ifeq ($(LZ4_TRAVIS_CI_ENV),-examples) -TRAVIS_TARGET=examples +ifneq (,$(filter test-%,$(LZ4_TRAVIS_CI_ENV))) +TRAVIS_TARGET=prg-travis else -TRAVIS_TARGET=test-prg +TRAVIS_TARGET=$(LZ4_TRAVIS_CI_ENV) endif @@ -113,7 +109,7 @@ liblz4: lz4.c lz4hc.c @echo compiling static library @$(CC) $(CPPFLAGS) $(CFLAGS) -c $^ @$(AR) rcs liblz4.a lz4.o lz4hc.o - @echo compiling dynamic library + @echo compiling dynamic library $(LIBVER) @$(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER) @echo creating versioned links @ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT_MAJOR) @@ -186,10 +182,10 @@ test-travis: $(TRAVIS_TARGET) cmake: @cd cmake_unofficial; cmake CMakeLists.txt; $(MAKE) -examples: +streaming-examples: cd examples; $(MAKE) -e test -test-prg: +prg-travis: @cd $(PRGDIR); $(MAKE) -e test-travis endif diff --git a/programs/Makefile b/programs/Makefile index ba08cfb..1d7e17f 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -58,29 +58,7 @@ endif # Select test target for Travis CI's Build Matrix -ifeq ($(LZ4_TRAVIS_CI_ENV),-lz4) -TRAVIS_TARGET=test-lz4 -else ifeq ($(LZ4_TRAVIS_CI_ENV),-lz4c) -TRAVIS_TARGET=test-lz4c -else ifeq ($(LZ4_TRAVIS_CI_ENV),-lz4c32) -TRAVIS_TARGET=test-lz4c32 -else ifeq ($(LZ4_TRAVIS_CI_ENV),-fullbench) -TRAVIS_TARGET=test-fullbench -else ifeq ($(LZ4_TRAVIS_CI_ENV),-fullbench32) -TRAVIS_TARGET=test-fullbench32 -else ifeq ($(LZ4_TRAVIS_CI_ENV),-fuzzer) -TRAVIS_TARGET=test-fuzzer -else ifeq ($(LZ4_TRAVIS_CI_ENV),-fuzzer32) -TRAVIS_TARGET=test-fuzzer32 -else ifeq ($(LZ4_TRAVIS_CI_ENV),-frametest) -TRAVIS_TARGET=test-frametest -else ifeq ($(LZ4_TRAVIS_CI_ENV),-frametest32) -TRAVIS_TARGET=test-frametest32 -else ifeq ($(LZ4_TRAVIS_CI_ENV),-mem) -TRAVIS_TARGET=test-mem -else -TRAVIS_TARGET=test -endif +TRAVIS_TARGET=$(LZ4_TRAVIS_CI_ENV) default: lz4 lz4c @@ -166,6 +144,7 @@ test-lz4: lz4 datagen ./datagen | ./lz4 | ./lz4 -vdq > $(VOID) ./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -vdq > $(VOID) ./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -vdq > $(VOID) +# test frame concatenation with null-length frame echo -n > empty.test echo hi > nonempty.test cat nonempty.test empty.test nonempty.test > orig.test @@ -175,6 +154,8 @@ test-lz4: lz4 datagen ./lz4 -d concat.lz4.test > result.test sdiff orig.test result.test @rm *.test + @echo frame concatenation test completed +# test frame concatenation with null-length frame test-lz4c: lz4c datagen -- cgit v0.12