From ffb27d4eca6bd80c068e745388faa31a0982115c Mon Sep 17 00:00:00 2001 From: "yann.collet.73@gmail.com" Date: Fri, 30 Nov 2012 13:23:36 +0000 Subject: =?UTF-8?q?LZ4=20HC=20:=20extended=20detection=20window.=20Thanks?= =?UTF-8?q?=20to=20Adrien=20Grand.=20Fuzzer=20:=20more=20tests=20cases=20l?= =?UTF-8?q?z4demo=20:=20detect=20write=20errors.=20Thanks=20to=20Dima=20Ti?= =?UTF-8?q?snek=20bench.c=20:=20compatibility=20with=20Solaris=2064.=20Tha?= =?UTF-8?q?nks=20to=20Thorbj=C3=B8rn=20Willoch=20LZ4=5FcompressBound()=20:?= =?UTF-8?q?=20now=20both=20in=20inline=20function=20and=20macro=20format.?= =?UTF-8?q?=20Thanks=20to=20Jacob=20Gorm=20Hansen?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit git-svn-id: https://lz4.googlecode.com/svn/trunk@84 650e7d94-2a16-8b24-b05c-7c0b3f6821cd --- bench.c | 2 +- fuzzer.c | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++-------- lz4.c | 3 +-- lz4.h | 15 +++++++++++++-- lz4demo.c | 19 ++++++++++++------- lz4hc.c | 4 ++-- 6 files changed, 80 insertions(+), 22 deletions(-) diff --git a/bench.c b/bench.c index b0a0ecb..015c36e 100644 --- a/bench.c +++ b/bench.c @@ -33,7 +33,7 @@ #if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions # define _LARGEFILE_SOURCE # define FILE_OFFSET_BITS=64 -#else +#elif ! defined(__LP64__) // No point defining Large file for 64 bit # define _LARGEFILE64_SOURCE #endif diff --git a/fuzzer.c b/fuzzer.c index 9fea8fd..9ba79d4 100644 --- a/fuzzer.c +++ b/fuzzer.c @@ -33,7 +33,7 @@ // Includes //************************************** #include -#include // fgets +#include // fgets, sscanf #include // timeb #include "lz4.h" @@ -97,11 +97,12 @@ int main() { unsigned long long bytes = 0; unsigned long long cbytes = 0; unsigned char buf[LEN]; -# define FUZ_max LZ4_compressBound(LEN) + unsigned char testOut[LEN+1]; +# define FUZ_max LZ4_COMPRESSBOUND(LEN) # define FUZ_avail ROUND_PAGE(FUZ_max) const int off_full = FUZ_avail - FUZ_max; unsigned char cbuf[FUZ_avail + PAGE_SIZE]; - unsigned int seed, cur_seq, seeds[NUM_SEQ], timestamp=FUZ_GetMilliStart(); + unsigned int seed, cur_seq=PRIME3, seeds[NUM_SEQ], timestamp=FUZ_GetMilliStart(); int i, j, k, ret, len; char userInput[30] = {0}; @@ -120,6 +121,7 @@ int main() { for (i = 0; i < NB_ATTEMPTS; i++) { printf("\r%7i /%7i\r", i, NB_ATTEMPTS); + FUZ_rand(&seed); for (j = 0; j < NUM_SEQ; j++) { seeds[j] = FUZ_rand(&seed) << 8; @@ -136,23 +138,64 @@ int main() { } buf[j] = FUZ_rand(&cur_seq) >> 16; } + + // Test compression ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[off_full], LEN, FUZ_max); + if (ret == 0) { printf("compression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; } len = ret; - // Test compression with output size being exactly what's necessary + // Test decoding with output size being exactly what's necessary => must work + ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN); + if (ret<0) { printf("decompression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test decoding with one byte missing => must fail + ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN-1); + if (ret>=0) { printf("decompression should have failed, due to Output Size being too small : seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test decoding with one byte too much => must fail + ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN+1); + if (ret>=0) { printf("decompression should have failed, due to Output Size being too large : seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test decoding with enough output size => must work + ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN+1); + if (ret<0) { printf("decompression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test decoding with output size being exactly what's necessary => should work + ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN); + if (ret<0) { printf("decompression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test decoding with output size being one byte too short => must fail + ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN-1); + if (ret>=0) { printf("decompression should have failed, due to Output Size being too small : seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test decoding with input size being one byte too short => must fail + ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len-1, LEN); + if (ret>=0) { printf("decompression should have failed, due to input size being too small : seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test decoding with input size being one byte too large => must fail + ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len+1, LEN); + if (ret>=0) { printf("decompression should have failed, due to input size being too large : seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test compression with output size being exactly what's necessary (should work) ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, len); - if (!test_canary(&cbuf[FUZ_avail])) { printf("compression overran output buffer: seed %u, len %d, olen %d\n", seed, LEN, len); return 1; } - if (ret == 0) { printf("compression failed despite sufficient space: seed %u, len %d\n", seed, LEN); return 1; } + if (!test_canary(&cbuf[FUZ_avail])) { printf("compression overran output buffer: seed %u, len %d, olen %d\n", seed, LEN, len); goto _output_error; } + if (ret == 0) { printf("compression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; } // Test compression with just one missing byte into output buffer => must fail ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-(len-1)], LEN, len-1); - if (ret) { printf("compression overran output buffer: seed %u, len %d, olen %d => ret %d", seed, LEN, len-1, ret); return 1; } - if (!test_canary(&cbuf[FUZ_avail])) { printf("compression overran output buffer: seed %u, len %d, olen %d", seed, LEN, len-1); return 1; } + if (ret) { printf("compression overran output buffer: seed %u, len %d, olen %d => ret %d", seed, LEN, len-1, ret); goto _output_error; } + if (!test_canary(&cbuf[FUZ_avail])) { printf("compression overran output buffer: seed %u, len %d, olen %d", seed, LEN, len-1); goto _output_error; } bytes += LEN; cbytes += len; } + printf("all tests completed successfully \n"); printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100); + getchar(); return 0; + +_output_error: + getchar(); + return 1; } diff --git a/lz4.c b/lz4.c index 75dc64a..a651748 100644 --- a/lz4.c +++ b/lz4.c @@ -91,7 +91,7 @@ //************************************** // Compiler Options //************************************** -#if __STDC_VERSION__ >= 199901L // C99 +#if __STDC_VERSION__ >= 199901L // C99 /* "restrict" is a known keyword */ #else # define restrict // Disable restrict @@ -100,7 +100,6 @@ #define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) #ifdef _MSC_VER // Visual Studio -# define inline __forceinline // Visual is not C99, but supports some kind of inline # include // For Visual 2005 # if LZ4_ARCH64 // 64-bit # pragma intrinsic(_BitScanForward64) // For Visual 2005 diff --git a/lz4.h b/lz4.h index dee3984..3680121 100644 --- a/lz4.h +++ b/lz4.h @@ -38,6 +38,14 @@ extern "C" { #endif +//************************************** +// Compiler Options +//************************************** +#ifdef _MSC_VER // Visual Studio +# define inline __inline // Visual is not C99, but supports some kind of inline +#endif + + //**************************** // Simple Functions //**************************** @@ -50,7 +58,7 @@ LZ4_compress() : Compresses 'isize' bytes from 'source' into 'dest'. Destination buffer must be already allocated, and must be sized to handle worst cases situations (input data not compressible) - Worst case size evaluation is provided by macro LZ4_compressBound() + Worst case size evaluation is provided by function LZ4_compressBound() isize : is the input size. Max supported value is ~1.9GB return : the number of bytes written in buffer dest @@ -70,12 +78,15 @@ LZ4_uncompress() : // Advanced Functions //**************************** -#define LZ4_compressBound(isize) (isize + (isize/255) + 16) +static inline int LZ4_compressBound(int isize) { return ((isize) + ((isize)/255) + 16); } +#define LZ4_COMPRESSBOUND( isize) ((isize) + ((isize)/255) + 16) /* LZ4_compressBound() : Provides the maximum size that LZ4 may output in a "worst case" scenario (input data not compressible) primarily useful for memory allocation of output buffer. + inline function is recommended for the general case, + but macro is also provided when results need to be evaluated at compile time (such as table size allocation). isize : is the input size. Max supported value is ~1.9GB return : maximum output size in a "worst case" scenario diff --git a/lz4demo.c b/lz4demo.c index 6367401..067ac2e 100644 --- a/lz4demo.c +++ b/lz4demo.c @@ -53,7 +53,7 @@ //************************************** -// Compiler functions +// Compiler-specific functions //************************************** #define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) @@ -91,9 +91,9 @@ static inline unsigned int swap32(unsigned int x) { // Architecture Macros //************************************** static const int one = 1; -#define CPU_LITTLE_ENDIAN (*(char*)(&one)) -#define CPU_BIG_ENDIAN (!CPU_LITTLE_ENDIAN) -#define LITTLE_ENDIAN32(i) if (CPU_BIG_ENDIAN) { i = swap32(i); } +#define CPU_LITTLE_ENDIAN (*(char*)(&one)) +#define CPU_BIG_ENDIAN (!CPU_LITTLE_ENDIAN) +#define LITTLE_ENDIAN32(i) if (CPU_BIG_ENDIAN) { i = swap32(i); } //************************************** @@ -177,6 +177,7 @@ int compress_file(char* input_filename, char* output_filename, int compressionle int r; int displayLevel = (compressionlevel>0); clock_t start, end; + size_t sizeCheck; // Init @@ -199,7 +200,8 @@ int compress_file(char* input_filename, char* output_filename, int compressionle u32var = ARCHIVE_MAGICNUMBER; LITTLE_ENDIAN32(u32var); *(unsigned int*)out_buff = u32var; - fwrite(out_buff, 1, ARCHIVE_MAGICNUMBER_SIZE, foutput); + sizeCheck = fwrite(out_buff, 1, ARCHIVE_MAGICNUMBER_SIZE, foutput); + if (sizeCheck!=ARCHIVE_MAGICNUMBER_SIZE) { DISPLAY("write error\n"); return 10; } // Main Loop while (1) @@ -220,7 +222,8 @@ int compress_file(char* input_filename, char* output_filename, int compressionle LITTLE_ENDIAN32(outSize); * (unsigned int*) out_buff = outSize; LITTLE_ENDIAN32(outSize); - fwrite(out_buff, 1, outSize+4, foutput); + sizeCheck = fwrite(out_buff, 1, outSize+4, foutput); + if (sizeCheck!=(size_t)(outSize+4)) { DISPLAY("write error\n"); return 11; } } // Status @@ -254,6 +257,7 @@ int decode_file(char* input_filename, char* output_filename) FILE* foutput; clock_t start, end; int r; + size_t sizeCheck; // Init @@ -291,7 +295,8 @@ int decode_file(char* input_filename, char* output_filename) filesize += sinkint; // Write Block - fwrite(out_buff, 1, sinkint, foutput); + sizeCheck = fwrite(out_buff, 1, sinkint, foutput); + if (sizeCheck != (size_t)sinkint) { DISPLAY("write error\n"); return 12; } } // Status diff --git a/lz4hc.c b/lz4hc.c index 2ab507e..7cd8bb4 100644 --- a/lz4hc.c +++ b/lz4hc.c @@ -337,7 +337,7 @@ inline static int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const // HC4 match finder LZ4HC_Insert(hc4, ip); ref = HASH_POINTER(ip); - while ((ref > (ip-MAX_DISTANCE)) && (nbAttempts)) + while ((ref >= (ip-MAX_DISTANCE)) && (nbAttempts)) { nbAttempts--; if (*(ref+ml) == *(ip+ml)) @@ -380,7 +380,7 @@ inline static int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const LZ4HC_Insert(hc4, ip); ref = HASH_POINTER(ip); - while ((ref > ip-MAX_DISTANCE) && (ref >= hc4->base) && (nbAttempts)) + while ((ref >= ip-MAX_DISTANCE) && (ref >= hc4->base) && (nbAttempts)) { nbAttempts--; if (*(startLimit + longest) == *(ref - delta + longest)) -- cgit v0.12