summaryrefslogtreecommitdiffstats
path: root/programs/fuzzer.c
diff options
context:
space:
mode:
authorYann Collet <yann.collet.73@gmail.com>2014-10-20 00:08:21 (GMT)
committerYann Collet <yann.collet.73@gmail.com>2014-10-20 00:08:21 (GMT)
commit3dab5f476a2e5a0cd4cd9a859e94a5110abda23d (patch)
tree17625ab18fb75e63bfa2ba65052e1c77d592a607 /programs/fuzzer.c
parente2a985f52444ad2e1438d2a71575185f31a08540 (diff)
downloadlz4-3dab5f476a2e5a0cd4cd9a859e94a5110abda23d.zip
lz4-3dab5f476a2e5a0cd4cd9a859e94a5110abda23d.tar.gz
lz4-3dab5f476a2e5a0cd4cd9a859e94a5110abda23d.tar.bz2
LZ4 HC : External Dictionary compression : First implementation
Diffstat (limited to 'programs/fuzzer.c')
-rw-r--r--programs/fuzzer.c100
1 files changed, 68 insertions, 32 deletions
diff --git a/programs/fuzzer.c b/programs/fuzzer.c
index cc73262..cf56251 100644
--- a/programs/fuzzer.c
+++ b/programs/fuzzer.c
@@ -263,7 +263,7 @@ _overflowError:
}
-static void FUZ_displayUpdate(int testNb)
+static void FUZ_displayUpdate(unsigned testNb)
{
if ((FUZ_GetMilliSpan(g_time) > g_refreshRate) || (g_displayLevel>=3))
{
@@ -274,7 +274,7 @@ static void FUZ_displayUpdate(int testNb)
}
-static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility)
+static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const double compressibility)
{
unsigned long long bytes = 0;
unsigned long long cbytes = 0;
@@ -283,15 +283,17 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili
void* CNBuffer;
char* compressedBuffer;
char* decodedBuffer;
-# define FUZ_max LZ4_COMPRESSBOUND(LEN)
- int ret, cycleNb;
-# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %i : ", testNb); printf(__VA_ARGS__); \
- printf(" (seed %u, cycle %i) \n", seed, cycleNb); goto _output_error; }
-# define FUZ_DISPLAYTEST { testNb++; g_displayLevel<3 ? 0 : printf("%2i\b\b", testNb); if (g_displayLevel==4) fflush(stdout); }
+# define FUZ_max LZ4_COMPRESSBOUND(LEN)
+ int ret;
+ unsigned cycleNb;
+# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \
+ printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; }
+# define FUZ_DISPLAYTEST { testNb++; g_displayLevel<3 ? 0 : printf("%2u\b\b", testNb); if (g_displayLevel==4) fflush(stdout); }
void* stateLZ4 = malloc(LZ4_sizeofState());
void* stateLZ4HC = malloc(LZ4_sizeofStateHC());
void* LZ4continue;
LZ4_stream_t LZ4dict;
+ LZ4_streamHC_t LZ4dictHC;
U32 crcOrig, crcCheck;
U32 coreRandState = seed;
U32 randState = coreRandState ^ PRIME3;
@@ -303,21 +305,19 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili
// Create compressible test buffer
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
- compressedBuffer = malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE));
- decodedBuffer = malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE);
+ compressedBuffer = (char*)malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE));
+ decodedBuffer = (char*)malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE);
// move to startCycle
for (cycleNb = 0; cycleNb < startCycle; cycleNb++)
{
- // synd rand & dict
- int dictSize, blockSize, blockStart;
- char* dict;
- char* block;
-
(void)FUZ_rand(&coreRandState);
if (0) // some problems related to dictionary re-use; in this case, enable this loop
{
+ int dictSize, blockSize, blockStart;
+ char* dict;
+ char* block;
FUZ_displayUpdate(cycleNb);
randState = coreRandState ^ PRIME3;
blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE;
@@ -338,7 +338,7 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili
// Test loop
for (cycleNb = startCycle; cycleNb < nbCycles; cycleNb++)
{
- int testNb = 0;
+ U32 testNb = 0;
char* dict;
char* block;
int dictSize, blockSize, blockStart, compressedSize, HCcompressedSize;
@@ -506,8 +506,8 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili
// Compress using dictionary
FUZ_DISPLAYTEST;
LZ4continue = LZ4_create (dict);
- LZ4_compress_continue (LZ4continue, dict, compressedBuffer, dictSize); // Just to fill hash tables
- blockContinueCompressedSize = LZ4_compress_continue (LZ4continue, block, compressedBuffer, blockSize);
+ LZ4_compress_continue ((LZ4_stream_t*)LZ4continue, dict, compressedBuffer, dictSize); // Just to fill hash tables
+ blockContinueCompressedSize = LZ4_compress_continue ((LZ4_stream_t*)LZ4continue, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
free (LZ4continue);
@@ -534,7 +534,7 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili
// Compress using External dictionary
FUZ_DISPLAYTEST;
- dict -= 9; // Separation, so it is an ExtDict
+ dict -= (FUZ_rand(&randState) & 0xF) + 1; // Separation, so it is an ExtDict
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
LZ4_loadDict(&LZ4dict, dict, dictSize);
blockContinueCompressedSize = LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
@@ -563,7 +563,6 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili
int i=0;
while (block[i]==decodedBuffer[i]) i++;
printf("Wrong Byte at position %i/%i\n", i, blockSize);
-
}
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
@@ -588,23 +587,60 @@ static int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibili
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
FUZ_DISPLAYTEST;
- if (blockSize > 10)
{
- decodedBuffer[blockSize-10] = 0;
- ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-10, dict, dictSize);
- FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-10 byte)");
- FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-10 byte) (blockSize=%i)", blockSize);
+ U32 missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
+ if ((U32)blockSize > missingBytes)
+ {
+ decodedBuffer[blockSize-missingBytes] = 0;
+ ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-missingBytes, dict, dictSize);
+ FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%u byte)", missingBytes);
+ FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize);
+ }
+ }
+
+ // Compress HC using External dictionary
+ FUZ_DISPLAYTEST;
+ dict -= (FUZ_rand(&randState) & 7); // even bigger separation
+ if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
+ LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
+ blockContinueCompressedSize = LZ4_compressHC_continue(&LZ4dictHC, block, compressedBuffer, blockSize);
+ FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compressHC_continue failed");
+
+ FUZ_DISPLAYTEST;
+ LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
+ ret = LZ4_compressHC_limitedOutput_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
+ FUZ_CHECKTEST(ret>0, "LZ4_compressHC_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer");
+
+ FUZ_DISPLAYTEST;
+ LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
+ ret = LZ4_compressHC_limitedOutput_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
+ FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
+ FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer");
+
+ FUZ_DISPLAYTEST;
+ decodedBuffer[blockSize] = 0;
+ ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
+ FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
+ FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size")
+ crcCheck = XXH32(decodedBuffer, blockSize, 0);
+ if (crcCheck!=crcOrig)
+ {
+ int i=0;
+ while (block[i]==decodedBuffer[i]) i++;
+ printf("Wrong Byte at position %i/%i\n", i, blockSize);
}
+ FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
- // Fill stats
+ // ***** End of tests *** //
+ // Fill stats
bytes += blockSize;
cbytes += compressedSize;
hcbytes += HCcompressedSize;
ccbytes += blockContinueCompressedSize;
}
- printf("\r%7i /%7i - ", cycleNb, nbCycles);
+ printf("\r%7u /%7u - ", cycleNb, nbCycles);
printf("all tests completed successfully \n");
printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100);
printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100);
@@ -627,8 +663,8 @@ _output_error:
}
}
-static const unsigned testInputSize = 128 KB;
-static const unsigned testCompressedSize = 64 KB;
+#define testInputSize (128 KB)
+#define testCompressedSize (64 KB)
static void FUZ_unitTests(void)
{
@@ -703,17 +739,17 @@ static void FUZ_unitTests(void)
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption");
}
- // remote dictionary compression test
+ // remote dictionary HC compression test
crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0);
LZ4_resetStreamHC(&sHC, 0);
LZ4_loadDictHC(&sHC, testInput, 32 KB);
result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1);
- //FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result);
+ FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result);
result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 32 KB);
- //FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() dictionary decompression failed");
+ FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe_usingDict() decompression failed following remote dictionary HC compression test");
crcNew = XXH64(testVerify, testCompressedSize, 0);
- //FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption");
+ FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() decompression corruption");
}
printf("All unit tests completed succesfully \n");