LZ4 HC : extended detection window. Thanks to Adrien Grand.

Fuzzer : more tests cases
lz4demo : detect write errors. Thanks to Dima Tisnek
bench.c : compatibility with Solaris 64. Thanks to Thorbjørn Willoch
LZ4_compressBound() : now both in inline function and macro format. Thanks to Jacob Gorm Hansen

git-svn-id: https://lz4.googlecode.com/svn/trunk@84 650e7d94-2a16-8b24-b05c-7c0b3f6821cd

6 files changed, 80 insertions, 22 deletions

diff --git a/bench.c b/bench.c
index b0a0ecb..015c36e 100644
--- a/bench.c
+++ b/bench.c
@@ -33,7 +33,7 @@
 #if (defined(__sun__) && (!defined(__LP64__)))   // Sun Solaris 32-bits requires specific definitions

 #  define _LARGEFILE_SOURCE 

 #  define FILE_OFFSET_BITS=64

-#else

+#elif ! defined(__LP64__)                        // No point defining Large file for 64 bit

 #  define _LARGEFILE64_SOURCE

 #endif

 

diff --git a/fuzzer.c b/fuzzer.c
index 9fea8fd..9ba79d4 100644
--- a/fuzzer.c
+++ b/fuzzer.c
@@ -33,7 +33,7 @@
 // Includes

 //**************************************

 #include <stdlib.h>

-#include <stdio.h>      // fgets

+#include <stdio.h>      // fgets, sscanf

 #include <sys/timeb.h>  // timeb

 #include "lz4.h"

 

@@ -97,11 +97,12 @@ int main() {
         unsigned long long bytes = 0;

         unsigned long long cbytes = 0;

         unsigned char buf[LEN];

-#       define FUZ_max   LZ4_compressBound(LEN)

+        unsigned char testOut[LEN+1];

+#       define FUZ_max   LZ4_COMPRESSBOUND(LEN)

 #       define FUZ_avail ROUND_PAGE(FUZ_max)

         const int off_full = FUZ_avail - FUZ_max;

         unsigned char cbuf[FUZ_avail + PAGE_SIZE];

-		unsigned int seed, cur_seq, seeds[NUM_SEQ], timestamp=FUZ_GetMilliStart();

+		unsigned int seed, cur_seq=PRIME3, seeds[NUM_SEQ], timestamp=FUZ_GetMilliStart();

         int i, j, k, ret, len;

 		char userInput[30] = {0};

 

@@ -120,6 +121,7 @@ int main() {
 

         for (i = 0; i < NB_ATTEMPTS; i++) {

 			printf("\r%7i /%7i\r", i, NB_ATTEMPTS);

+			

 			FUZ_rand(&seed);

             for (j = 0; j < NUM_SEQ; j++) {

                     seeds[j] = FUZ_rand(&seed) << 8;

@@ -136,23 +138,64 @@ int main() {
                     }

                     buf[j] = FUZ_rand(&cur_seq) >> 16;

             }

+

+			// Test compression

             ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[off_full], LEN, FUZ_max);

+			if (ret == 0) { printf("compression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; }

             len = ret;

 

-			// Test compression with output size being exactly what's necessary

+			// Test decoding with output size being exactly what's necessary => must work

+			ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN);

+			if (ret<0) { printf("decompression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; }

+

+			// Test decoding with one byte missing => must fail

+			ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN-1);

+			if (ret>=0) { printf("decompression should have failed, due to Output Size being too small : seed %u, len %d\n", seed, LEN); goto _output_error; }

+

+			// Test decoding with one byte too much => must fail

+			ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN+1);

+			if (ret>=0) { printf("decompression should have failed, due to Output Size being too large : seed %u, len %d\n", seed, LEN); goto _output_error; }

+

+			// Test decoding with enough output size => must work

+			ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN+1);

+			if (ret<0) { printf("decompression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; }

+

+			// Test decoding with output size being exactly what's necessary => should work

+			ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN);

+			if (ret<0) { printf("decompression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; }

+

+			// Test decoding with output size being one byte too short => must fail

+			ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN-1);

+			if (ret>=0) { printf("decompression should have failed, due to Output Size being too small : seed %u, len %d\n", seed, LEN); goto _output_error; }

+

+			// Test decoding with input size being one byte too short => must fail

+			ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len-1, LEN);

+			if (ret>=0) { printf("decompression should have failed, due to input size being too small : seed %u, len %d\n", seed, LEN); goto _output_error; }

+

+			// Test decoding with input size being one byte too large => must fail

+			ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len+1, LEN);

+			if (ret>=0) { printf("decompression should have failed, due to input size being too large : seed %u, len %d\n", seed, LEN); goto _output_error; }

+

+			// Test compression with output size being exactly what's necessary (should work)

             ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, len);

-            if (!test_canary(&cbuf[FUZ_avail])) { printf("compression overran output buffer: seed %u, len %d, olen %d\n", seed, LEN, len); return 1; }

-            if (ret == 0) { printf("compression failed despite sufficient space: seed %u, len %d\n", seed, LEN); return 1; }

+            if (!test_canary(&cbuf[FUZ_avail])) { printf("compression overran output buffer: seed %u, len %d, olen %d\n", seed, LEN, len); goto _output_error; }

+            if (ret == 0) { printf("compression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; }

 

 			// Test compression with just one missing byte into output buffer => must fail

             ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-(len-1)], LEN, len-1);

-            if (ret) { printf("compression overran output buffer: seed %u, len %d, olen %d => ret %d", seed, LEN, len-1, ret); return 1; }

-            if (!test_canary(&cbuf[FUZ_avail])) { printf("compression overran output buffer: seed %u, len %d, olen %d", seed, LEN, len-1); return 1; }

+            if (ret) { printf("compression overran output buffer: seed %u, len %d, olen %d => ret %d", seed, LEN, len-1, ret); goto _output_error; }

+            if (!test_canary(&cbuf[FUZ_avail])) { printf("compression overran output buffer: seed %u, len %d, olen %d", seed, LEN, len-1); goto _output_error; }

 

 			bytes += LEN;

             cbytes += len;

         }

+

 		printf("all tests completed successfully \n");

         printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100);

+		getchar();

         return 0;

+

+_output_error:

+		getchar();

+		return 1;

 }

diff --git a/lz4.c b/lz4.c
index 75dc64a..a651748 100644
--- a/lz4.c
+++ b/lz4.c
@@ -91,7 +91,7 @@
 //**************************************

 // Compiler Options

 //**************************************

-#if __STDC_VERSION__ >= 199901L // C99

+#if __STDC_VERSION__ >= 199901L   // C99

 /* "restrict" is a known keyword */

 #else

 #  define restrict // Disable restrict

@@ -100,7 +100,6 @@
 #define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)

 

 #ifdef _MSC_VER  // Visual Studio

-#  define inline __forceinline           // Visual is not C99, but supports some kind of inline

 #  include <intrin.h>   // For Visual 2005

 #  if LZ4_ARCH64	// 64-bit

 #    pragma intrinsic(_BitScanForward64) // For Visual 2005

diff --git a/lz4.h b/lz4.h
index dee3984..3680121 100644
--- a/lz4.h
+++ b/lz4.h
@@ -38,6 +38,14 @@ extern "C" {
 #endif

 

 

+//**************************************

+// Compiler Options

+//**************************************

+#ifdef _MSC_VER   // Visual Studio

+#  define inline __inline           // Visual is not C99, but supports some kind of inline

+#endif

+

+

 //****************************

 // Simple Functions

 //****************************

@@ -50,7 +58,7 @@ LZ4_compress() :
     Compresses 'isize' bytes from 'source' into 'dest'.

     Destination buffer must be already allocated,

     and must be sized to handle worst cases situations (input data not compressible)

-    Worst case size evaluation is provided by macro LZ4_compressBound()

+    Worst case size evaluation is provided by function LZ4_compressBound()

 

     isize  : is the input size. Max supported value is ~1.9GB

     return : the number of bytes written in buffer dest

@@ -70,12 +78,15 @@ LZ4_uncompress() :
 // Advanced Functions

 //****************************

 

-#define LZ4_compressBound(isize)   (isize + (isize/255) + 16)

+static inline int LZ4_compressBound(int isize)   { return ((isize) + ((isize)/255) + 16); }

+#define           LZ4_COMPRESSBOUND(    isize)            ((isize) + ((isize)/255) + 16)

 

 /*

 LZ4_compressBound() :

     Provides the maximum size that LZ4 may output in a "worst case" scenario (input data not compressible)

     primarily useful for memory allocation of output buffer.

+	inline function is recommended for the general case,

+	but macro is also provided when results need to be evaluated at compile time (such as table size allocation).

 

     isize  : is the input size. Max supported value is ~1.9GB

     return : maximum output size in a "worst case" scenario

diff --git a/lz4demo.c b/lz4demo.c
index 6367401..067ac2e 100644
--- a/lz4demo.c
+++ b/lz4demo.c
@@ -53,7 +53,7 @@
 

 

 //**************************************

-// Compiler functions

+// Compiler-specific functions

 //**************************************

 #define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)

 

@@ -91,9 +91,9 @@ static inline unsigned int swap32(unsigned int x) {
 // Architecture Macros

 //**************************************

 static const int one = 1;

-#define CPU_LITTLE_ENDIAN (*(char*)(&one))

-#define CPU_BIG_ENDIAN (!CPU_LITTLE_ENDIAN)

-#define LITTLE_ENDIAN32(i)   if (CPU_BIG_ENDIAN) { i = swap32(i); }

+#define CPU_LITTLE_ENDIAN  (*(char*)(&one))

+#define CPU_BIG_ENDIAN     (!CPU_LITTLE_ENDIAN)

+#define LITTLE_ENDIAN32(i) if (CPU_BIG_ENDIAN) { i = swap32(i); }

 

 

 //**************************************

@@ -177,6 +177,7 @@ int compress_file(char* input_filename, char* output_filename, int compressionle
 	int r;

 	int displayLevel = (compressionlevel>0);

 	clock_t start, end;

+	size_t sizeCheck;

 

 

 	// Init

@@ -199,7 +200,8 @@ int compress_file(char* input_filename, char* output_filename, int compressionle
 	u32var = ARCHIVE_MAGICNUMBER;

 	LITTLE_ENDIAN32(u32var);

 	*(unsigned int*)out_buff = u32var;

-	fwrite(out_buff, 1, ARCHIVE_MAGICNUMBER_SIZE, foutput);

+	sizeCheck = fwrite(out_buff, 1, ARCHIVE_MAGICNUMBER_SIZE, foutput);

+	if (sizeCheck!=ARCHIVE_MAGICNUMBER_SIZE) { DISPLAY("write error\n"); return 10; }

 

 	// Main Loop

 	while (1)

@@ -220,7 +222,8 @@ int compress_file(char* input_filename, char* output_filename, int compressionle
 		LITTLE_ENDIAN32(outSize);

 		* (unsigned int*) out_buff = outSize;

 		LITTLE_ENDIAN32(outSize);

-		fwrite(out_buff, 1, outSize+4, foutput);

+		sizeCheck = fwrite(out_buff, 1, outSize+4, foutput);

+		if (sizeCheck!=(size_t)(outSize+4)) { DISPLAY("write error\n"); return 11; }

 	}

 

 	// Status

@@ -254,6 +257,7 @@ int decode_file(char* input_filename, char* output_filename)
 	FILE* foutput;

 	clock_t start, end;

 	int r;

+	size_t sizeCheck;

 

 

 	// Init

@@ -291,7 +295,8 @@ int decode_file(char* input_filename, char* output_filename)
 		filesize += sinkint;

 

 		// Write Block

-		fwrite(out_buff, 1, sinkint, foutput);

+		sizeCheck = fwrite(out_buff, 1, sinkint, foutput);

+		if (sizeCheck != (size_t)sinkint) { DISPLAY("write error\n"); return 12; }

 	}

 

 	// Status

diff --git a/lz4hc.c b/lz4hc.c
index 2ab507e..7cd8bb4 100644
--- a/lz4hc.c
+++ b/lz4hc.c
@@ -337,7 +337,7 @@ inline static int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const
 	// HC4 match finder

 	LZ4HC_Insert(hc4, ip);

 	ref = HASH_POINTER(ip);

-	while ((ref > (ip-MAX_DISTANCE)) && (nbAttempts))

+	while ((ref >= (ip-MAX_DISTANCE)) && (nbAttempts))

 	{

 		nbAttempts--;

 		if (*(ref+ml) == *(ip+ml))

@@ -380,7 +380,7 @@ inline static int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const
 	LZ4HC_Insert(hc4, ip);

 	ref = HASH_POINTER(ip);

 

-	while ((ref > ip-MAX_DISTANCE) && (ref >= hc4->base) && (nbAttempts))

+	while ((ref >= ip-MAX_DISTANCE) && (ref >= hc4->base) && (nbAttempts))

 	{

 		nbAttempts--;

 		if (*(startLimit + longest) == *(ref - delta + longest))