summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorYann Collet <yann.collet.73@gmail.com>2014-09-05 15:32:04 (GMT)
committerYann Collet <yann.collet.73@gmail.com>2014-09-05 15:32:04 (GMT)
commitfd8665320fe03f9f61d438a3cc81502406859c2a (patch)
tree079f4dfccd585712f98f6ff1d934cb8f1ec38258
parent28f38c328d0974c1020e2c3a5a2ba9e085da5280 (diff)
downloadlz4-fd8665320fe03f9f61d438a3cc81502406859c2a.zip
lz4-fd8665320fe03f9f61d438a3cc81502406859c2a.tar.gz
lz4-fd8665320fe03f9f61d438a3cc81502406859c2a.tar.bz2
Added some more tests
-rw-r--r--lz4frame.c2
-rw-r--r--programs/frametest.c186
2 files changed, 152 insertions, 36 deletions
diff --git a/lz4frame.c b/lz4frame.c
index ef9c053..ec2af04 100644
--- a/lz4frame.c
+++ b/lz4frame.c
@@ -105,7 +105,7 @@
#define LZ4F_MAGICNUMBER 0x184D2204U
#define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U
-#define LZ4F_MAXHEADERFRAME_SIZE 19
+#define LZ4F_MAXHEADERFRAME_SIZE 7
#define LZ4F_BLOCKSIZEID_DEFAULT 4
diff --git a/programs/frametest.c b/programs/frametest.c
index 7742110..7d26bee 100644
--- a/programs/frametest.c
+++ b/programs/frametest.c
@@ -45,6 +45,7 @@
#include <sys/timeb.h> // timeb
#include <string.h> // strcmp
#include "lz4frame.h"
+#include "xxhash.h" // XXH64
/**************************************
@@ -91,8 +92,10 @@
/**************************************
Macros
**************************************/
-#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
-#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+#define DISPLAYUPDATE(l, ...) if (displayLevel>=l) { if (FUZ_GetMilliStart() > g_time + 200) { g_time = FUZ_GetMilliStart(); DISPLAY(__VA_ARGS__); } }
+static int g_time = 0;
/*****************************************
@@ -100,7 +103,7 @@
*****************************************/
static int no_prompt = 0;
static char* programName;
-static int displayLevel = 2;
+static int displayLevel = 3;
/*********************************************************
@@ -134,12 +137,12 @@ unsigned int FUZ_rand(unsigned int* src)
rand32 += PRIME2;
rand32 = FUZ_rotl32(rand32, 13);
*src = rand32;
- return rand32 >> 3;
+ return rand32 >> 5;
}
-#define FUZ_RAND15BITS ((FUZ_rand(seed) >> 3) & 32767)
-#define FUZ_RANDLENGTH ( ((FUZ_rand(seed) >> 7) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15)
+#define FUZ_RAND15BITS (FUZ_rand(seed) & 0x7FFF)
+#define FUZ_RANDLENGTH ( (FUZ_rand(seed) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15)
static void FUZ_fillCompressibleNoiseBuffer(void* buffer, unsigned bufferSize, double proba, U32* seed)
{
BYTE* BBuffer = (BYTE*)buffer;
@@ -177,8 +180,16 @@ static void FUZ_fillCompressibleNoiseBuffer(void* buffer, unsigned bufferSize, d
}
+static unsigned FUZ_highbit(U32 v32)
+{
+ unsigned nbBits = 0;
+ if (v32==0) return 0;
+ while (v32) { v32 >>= 1; nbBits ++; }
+ return nbBits;
+}
-int frameTest(U32 seed, int nbCycles, int startCycle, double compressibility)
+
+int basicTests(U32 seed, int nbCycles, int startCycle, double compressibility)
{
int testResult = 0;
void* CNBuffer;
@@ -188,22 +199,24 @@ int frameTest(U32 seed, int nbCycles, int startCycle, double compressibility)
size_t cSize, testSize;
LZ4F_preferences_t prefs = { 0 };
LZ4F_decompressionContext_t dCtx;
+ U64 crcOrig;
(void)nbCycles; (void)startCycle;
// Create compressible test buffer
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
- FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
compressedBuffer = malloc(LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL));
decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
+ FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
+ crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
// Trivial tests : one-step frame
testSize = COMPRESSIBLE_NOISE_LENGTH;
- DISPLAY("Using NULL preferences : \n");
+ DISPLAYLEVEL(3, "Using NULL preferences : \n");
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL);
if (LZ4F_isError(cSize)) goto _output_error;
- DISPLAY("Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
+ DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
- DISPLAY("Decompression test : \n");
+ DISPLAYLEVEL(3, "Decompression test : \n");
{
size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
size_t compressedBufferSize = cSize;
@@ -211,15 +224,18 @@ int frameTest(U32 seed, int nbCycles, int startCycle, double compressibility)
BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
BYTE* ip = (BYTE*)compressedBuffer;
BYTE* const iend = (BYTE*)compressedBuffer + cSize;
- LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx);
+ U64 crcDest;
+
+ LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
if (LZ4F_isError(errorCode)) goto _output_error;
- DISPLAY("Single Block : \n");
+ DISPLAYLEVEL(3, "Single Block : \n");
errorCode = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL);
- if (LZ4F_isError(errorCode)) goto _output_error;
- DISPLAY("Regenerated %i bytes \n", (int)decodedBufferSize);
+ crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
+ if (crcDest != crcOrig) goto _output_error;
+ DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize);
- DISPLAY("Byte after byte : \n");
+ DISPLAYLEVEL(3, "Byte after byte : \n");
while (ip < iend)
{
size_t oSize = oend-op;
@@ -230,63 +246,99 @@ int frameTest(U32 seed, int nbCycles, int startCycle, double compressibility)
op += oSize;
ip += iSize;
}
- DISPLAY("Regenerated %i bytes \n", (int)decodedBufferSize);
+ crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
+ if (crcDest != crcOrig) goto _output_error;
+ DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize);
errorCode = LZ4F_freeDecompressionContext(dCtx);
if (LZ4F_isError(errorCode)) goto _output_error;
}
- DISPLAY("Using 64 KB block : \n");
+ DISPLAYLEVEL(3, "Using 64 KB block : \n");
prefs.frameInfo.blockSizeID = max64KB;
+ prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
- DISPLAY("Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
+ DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
- DISPLAY("without checksum : \n");
+ DISPLAYLEVEL(3, "without checksum : \n");
prefs.frameInfo.contentChecksumFlag = noContentChecksum;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
- DISPLAY("Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
+ DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
- DISPLAY("Using 256 KB block : \n");
+ DISPLAYLEVEL(3, "Using 256 KB block : \n");
prefs.frameInfo.blockSizeID = max256KB;
prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
- DISPLAY("Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
+ DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
+
+ DISPLAYLEVEL(3, "Decompression test : \n");
+ {
+ size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
+ unsigned maxBits = FUZ_highbit(decodedBufferSize);
+ BYTE* op = (BYTE*)decodedBuffer;
+ BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
+ BYTE* ip = (BYTE*)compressedBuffer;
+ BYTE* const iend = (BYTE*)compressedBuffer + cSize;
+ U64 crcDest;
+
+ LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
+ if (LZ4F_isError(errorCode)) goto _output_error;
+
+ DISPLAYLEVEL(3, "random segment sizes : \n");
+ while (ip < iend)
+ {
+ unsigned nbBits = FUZ_rand(&randState) % maxBits;
+ size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
+ size_t oSize = oend-op;
+ if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
+ //DISPLAY("%7i : + %6i\n", (int)(ip-(BYTE*)compressedBuffer), (int)iSize);
+ errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
+ if (LZ4F_isError(errorCode)) goto _output_error;
+ op += oSize;
+ ip += iSize;
+ }
+ crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
+ if (crcDest != crcOrig) goto _output_error;
+ DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize);
+
+ errorCode = LZ4F_freeDecompressionContext(dCtx);
+ if (LZ4F_isError(errorCode)) goto _output_error;
+ }
- DISPLAY("without checksum : \n");
+ DISPLAYLEVEL(3, "without checksum : \n");
prefs.frameInfo.contentChecksumFlag = noContentChecksum;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
- DISPLAY("Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
+ DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
- DISPLAY("Using 1 MB block : \n");
+ DISPLAYLEVEL(3, "Using 1 MB block : \n");
prefs.frameInfo.blockSizeID = max1MB;
prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
- DISPLAY("Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
+ DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
- DISPLAY("without checksum : \n");
+ DISPLAYLEVEL(3, "without checksum : \n");
prefs.frameInfo.contentChecksumFlag = noContentChecksum;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
- DISPLAY("Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
+ DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
- DISPLAY("Using 4 MB block : \n");
+ DISPLAYLEVEL(3, "Using 4 MB block : \n");
prefs.frameInfo.blockSizeID = max4MB;
prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
- DISPLAY("Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
+ DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
- DISPLAY("without checksum : \n");
+ DISPLAYLEVEL(3, "without checksum : \n");
prefs.frameInfo.contentChecksumFlag = noContentChecksum;
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs);
if (LZ4F_isError(cSize)) goto _output_error;
- DISPLAY("Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
-
+ DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
_end:
free(CNBuffer);
@@ -302,6 +354,65 @@ _output_error:
}
+static const U32 srcDataLength = 4 MB;
+
+int fuzzerTests(U32 seed, unsigned nbTests, int startCycle, double compressibility)
+{
+ unsigned testResult = 0;
+ unsigned testNb = 0;
+ void* srcBuffer;
+ void* compressedBuffer;
+ void* decodedBuffer;
+ U32 randState = seed;
+ LZ4F_preferences_t prefs = { 0 };
+ LZ4F_decompressionContext_t dCtx;
+
+ (void)nbTests; (void)startCycle; (void)prefs;
+ // Create compressible test buffer
+ LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
+ srcBuffer = malloc(srcDataLength);
+ compressedBuffer = malloc(LZ4F_compressFrameBound(srcDataLength, NULL));
+ decodedBuffer = malloc(srcDataLength);
+ FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &randState);
+
+ // Select components of compression test
+ for (testNb=0; testNb < nbTests; testNb++)
+ {
+ unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(srcDataLength-1) - 1)) + 1;
+ size_t srcSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
+ size_t srcStart = FUZ_rand(&randState) % (srcDataLength - srcSize);
+ size_t cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(srcSize, NULL), srcBuffer+srcStart, srcSize, NULL);
+ U64 crcOrig = XXH64(srcBuffer+srcStart, srcSize, 1);
+ U64 crcRes;
+ LZ4F_errorCode_t err;
+ size_t outSize;
+ DISPLAYUPDATE(2, "%5i \r", testNb);
+ //DISPLAYLEVEL(2, "%5i \n", testNb);
+ if (LZ4F_isError(cSize)) goto _output_error;
+ outSize = srcDataLength;
+ err = LZ4F_decompress(dCtx, decodedBuffer, &outSize, compressedBuffer, &cSize, NULL);
+ if (LZ4F_isError(err)) goto _output_error;
+ if (outSize != srcSize) goto _output_error;
+ crcRes = XXH64(decodedBuffer, srcSize, 1);
+ if (crcRes != crcOrig) goto _output_error;
+ }
+
+ DISPLAYLEVEL(2, "All tests completed \n");
+
+_end:
+ free(srcBuffer);
+ free(compressedBuffer);
+ free(decodedBuffer);
+ return testResult;
+
+_output_error:
+ testResult = 1;
+ DISPLAY("Error detected ! \n");
+ if(!no_prompt) getchar();
+ goto _end;
+}
+
+
int FUZ_usage(void)
{
DISPLAY( "Usage :\n");
@@ -351,6 +462,10 @@ int main(int argc, char** argv)
argument++;
displayLevel=4;
break;
+ case 'q':
+ argument++;
+ displayLevel--;
+ break;
case 'i':
argument++;
nbTests=0;
@@ -408,5 +523,6 @@ int main(int argc, char** argv)
if (nbTests<=0) nbTests=1;
- return frameTest(seed, nbTests, testNb, ((double)proba) / 100);
+ basicTests(seed, nbTests, testNb, ((double)proba) / 100);
+ return fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
}