From fd8665320fe03f9f61d438a3cc81502406859c2a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 5 Sep 2014 16:32:04 +0100 Subject: Added some more tests --- lz4frame.c | 2 +- programs/frametest.c | 186 +++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 152 insertions(+), 36 deletions(-) diff --git a/lz4frame.c b/lz4frame.c index ef9c053..ec2af04 100644 --- a/lz4frame.c +++ b/lz4frame.c @@ -105,7 +105,7 @@ #define LZ4F_MAGICNUMBER 0x184D2204U #define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U -#define LZ4F_MAXHEADERFRAME_SIZE 19 +#define LZ4F_MAXHEADERFRAME_SIZE 7 #define LZ4F_BLOCKSIZEID_DEFAULT 4 diff --git a/programs/frametest.c b/programs/frametest.c index 7742110..7d26bee 100644 --- a/programs/frametest.c +++ b/programs/frametest.c @@ -45,6 +45,7 @@ #include // timeb #include // strcmp #include "lz4frame.h" +#include "xxhash.h" // XXH64 /************************************** @@ -91,8 +92,10 @@ /************************************** Macros **************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } +#define DISPLAYUPDATE(l, ...) if (displayLevel>=l) { if (FUZ_GetMilliStart() > g_time + 200) { g_time = FUZ_GetMilliStart(); DISPLAY(__VA_ARGS__); } } +static int g_time = 0; /***************************************** @@ -100,7 +103,7 @@ *****************************************/ static int no_prompt = 0; static char* programName; -static int displayLevel = 2; +static int displayLevel = 3; /********************************************************* @@ -134,12 +137,12 @@ unsigned int FUZ_rand(unsigned int* src) rand32 += PRIME2; rand32 = FUZ_rotl32(rand32, 13); *src = rand32; - return rand32 >> 3; + return rand32 >> 5; } -#define FUZ_RAND15BITS ((FUZ_rand(seed) >> 3) & 32767) -#define FUZ_RANDLENGTH ( ((FUZ_rand(seed) >> 7) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15) +#define FUZ_RAND15BITS (FUZ_rand(seed) & 0x7FFF) +#define FUZ_RANDLENGTH ( (FUZ_rand(seed) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15) static void FUZ_fillCompressibleNoiseBuffer(void* buffer, unsigned bufferSize, double proba, U32* seed) { BYTE* BBuffer = (BYTE*)buffer; @@ -177,8 +180,16 @@ static void FUZ_fillCompressibleNoiseBuffer(void* buffer, unsigned bufferSize, d } +static unsigned FUZ_highbit(U32 v32) +{ + unsigned nbBits = 0; + if (v32==0) return 0; + while (v32) { v32 >>= 1; nbBits ++; } + return nbBits; +} -int frameTest(U32 seed, int nbCycles, int startCycle, double compressibility) + +int basicTests(U32 seed, int nbCycles, int startCycle, double compressibility) { int testResult = 0; void* CNBuffer; @@ -188,22 +199,24 @@ int frameTest(U32 seed, int nbCycles, int startCycle, double compressibility) size_t cSize, testSize; LZ4F_preferences_t prefs = { 0 }; LZ4F_decompressionContext_t dCtx; + U64 crcOrig; (void)nbCycles; (void)startCycle; // Create compressible test buffer CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); - FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); compressedBuffer = malloc(LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL)); decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); + FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); + crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); // Trivial tests : one-step frame testSize = COMPRESSIBLE_NOISE_LENGTH; - DISPLAY("Using NULL preferences : \n"); + DISPLAYLEVEL(3, "Using NULL preferences : \n"); cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL); if (LZ4F_isError(cSize)) goto _output_error; - DISPLAY("Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); + DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - DISPLAY("Decompression test : \n"); + DISPLAYLEVEL(3, "Decompression test : \n"); { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; size_t compressedBufferSize = cSize; @@ -211,15 +224,18 @@ int frameTest(U32 seed, int nbCycles, int startCycle, double compressibility) BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; BYTE* ip = (BYTE*)compressedBuffer; BYTE* const iend = (BYTE*)compressedBuffer + cSize; - LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx); + U64 crcDest; + + LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); if (LZ4F_isError(errorCode)) goto _output_error; - DISPLAY("Single Block : \n"); + DISPLAYLEVEL(3, "Single Block : \n"); errorCode = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; - DISPLAY("Regenerated %i bytes \n", (int)decodedBufferSize); + crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); + if (crcDest != crcOrig) goto _output_error; + DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize); - DISPLAY("Byte after byte : \n"); + DISPLAYLEVEL(3, "Byte after byte : \n"); while (ip < iend) { size_t oSize = oend-op; @@ -230,63 +246,99 @@ int frameTest(U32 seed, int nbCycles, int startCycle, double compressibility) op += oSize; ip += iSize; } - DISPLAY("Regenerated %i bytes \n", (int)decodedBufferSize); + crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); + if (crcDest != crcOrig) goto _output_error; + DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize); errorCode = LZ4F_freeDecompressionContext(dCtx); if (LZ4F_isError(errorCode)) goto _output_error; } - DISPLAY("Using 64 KB block : \n"); + DISPLAYLEVEL(3, "Using 64 KB block : \n"); prefs.frameInfo.blockSizeID = max64KB; + prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled; cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; - DISPLAY("Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); + DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - DISPLAY("without checksum : \n"); + DISPLAYLEVEL(3, "without checksum : \n"); prefs.frameInfo.contentChecksumFlag = noContentChecksum; cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; - DISPLAY("Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); + DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - DISPLAY("Using 256 KB block : \n"); + DISPLAYLEVEL(3, "Using 256 KB block : \n"); prefs.frameInfo.blockSizeID = max256KB; prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled; cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; - DISPLAY("Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); + DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); + + DISPLAYLEVEL(3, "Decompression test : \n"); + { + size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; + unsigned maxBits = FUZ_highbit(decodedBufferSize); + BYTE* op = (BYTE*)decodedBuffer; + BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; + BYTE* ip = (BYTE*)compressedBuffer; + BYTE* const iend = (BYTE*)compressedBuffer + cSize; + U64 crcDest; + + LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); + if (LZ4F_isError(errorCode)) goto _output_error; + + DISPLAYLEVEL(3, "random segment sizes : \n"); + while (ip < iend) + { + unsigned nbBits = FUZ_rand(&randState) % maxBits; + size_t iSize = (FUZ_rand(&randState) & ((1< (size_t)(iend-ip)) iSize = iend-ip; + //DISPLAY("%7i : + %6i\n", (int)(ip-(BYTE*)compressedBuffer), (int)iSize); + errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); + if (LZ4F_isError(errorCode)) goto _output_error; + op += oSize; + ip += iSize; + } + crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); + if (crcDest != crcOrig) goto _output_error; + DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize); + + errorCode = LZ4F_freeDecompressionContext(dCtx); + if (LZ4F_isError(errorCode)) goto _output_error; + } - DISPLAY("without checksum : \n"); + DISPLAYLEVEL(3, "without checksum : \n"); prefs.frameInfo.contentChecksumFlag = noContentChecksum; cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; - DISPLAY("Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); + DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - DISPLAY("Using 1 MB block : \n"); + DISPLAYLEVEL(3, "Using 1 MB block : \n"); prefs.frameInfo.blockSizeID = max1MB; prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled; cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; - DISPLAY("Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); + DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - DISPLAY("without checksum : \n"); + DISPLAYLEVEL(3, "without checksum : \n"); prefs.frameInfo.contentChecksumFlag = noContentChecksum; cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; - DISPLAY("Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); + DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - DISPLAY("Using 4 MB block : \n"); + DISPLAYLEVEL(3, "Using 4 MB block : \n"); prefs.frameInfo.blockSizeID = max4MB; prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled; cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; - DISPLAY("Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); + DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - DISPLAY("without checksum : \n"); + DISPLAYLEVEL(3, "without checksum : \n"); prefs.frameInfo.contentChecksumFlag = noContentChecksum; cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &(prefs.frameInfo)), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; - DISPLAY("Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - + DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); _end: free(CNBuffer); @@ -302,6 +354,65 @@ _output_error: } +static const U32 srcDataLength = 4 MB; + +int fuzzerTests(U32 seed, unsigned nbTests, int startCycle, double compressibility) +{ + unsigned testResult = 0; + unsigned testNb = 0; + void* srcBuffer; + void* compressedBuffer; + void* decodedBuffer; + U32 randState = seed; + LZ4F_preferences_t prefs = { 0 }; + LZ4F_decompressionContext_t dCtx; + + (void)nbTests; (void)startCycle; (void)prefs; + // Create compressible test buffer + LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); + srcBuffer = malloc(srcDataLength); + compressedBuffer = malloc(LZ4F_compressFrameBound(srcDataLength, NULL)); + decodedBuffer = malloc(srcDataLength); + FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &randState); + + // Select components of compression test + for (testNb=0; testNb < nbTests; testNb++) + { + unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(srcDataLength-1) - 1)) + 1; + size_t srcSize = (FUZ_rand(&randState) & ((1<