From d2d566097031e5119980eb5ebbfa47023bc33c55 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 4 Sep 2018 16:50:34 -0700 Subject: new test program : roundTripTest make a round trip test with arbitrary input file, generate an `abort()` on error, to work in tandem with `afl`. note : currently locked on level 9, to investigate #560. --- tests/.gitignore | 1 + tests/Makefile | 5 +- tests/roundTripTest.c | 203 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 tests/roundTripTest.c diff --git a/tests/.gitignore b/tests/.gitignore index 36dff42..58947f7 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -8,6 +8,7 @@ fullbench32 fuzzer fuzzer32 fasttest +roundTripTest checkTag # test artefacts diff --git a/tests/Makefile b/tests/Makefile index 81033b5..bc43234 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -63,7 +63,7 @@ NB_LOOPS ?= -i1 default: all -all: fullbench fuzzer frametest datagen +all: fullbench fuzzer frametest roundTripTest datagen all32: CFLAGS+=-m32 all32: all @@ -103,6 +103,9 @@ fuzzer : lz4.o lz4hc.o xxhash.o fuzzer.c frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c $(CC) $(FLAGS) $^ -o $@$(EXT) +roundTripTest : lz4.o lz4hc.o xxhash.o roundTripTest.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + datagen : $(PRGDIR)/datagen.c datagencli.c $(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT) diff --git a/tests/roundTripTest.c b/tests/roundTripTest.c new file mode 100644 index 0000000..2f161d8 --- /dev/null +++ b/tests/roundTripTest.c @@ -0,0 +1,203 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* + * This program takes a file in input, + * performs an LZ4 round-trip test (compression - decompress) + * compares the result with original + * and generates an abort() on corruption detection, + * in order for afl to register the event as a crash. +*/ + + +/*=========================================== +* Dependencies +*==========================================*/ +#include /* size_t */ +#include /* malloc, free, exit */ +#include /* fprintf */ +#include /* strcmp */ +#include +#include /* stat */ +#include /* stat */ +#include "xxhash.h" + +#include "lz4.h" +#include "lz4hc.h" + + +/*=========================================== +* Macros +*==========================================*/ +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + +#define MSG(...) fprintf(stderr, __VA_ARGS__) + +#define CONTROL_MSG(c, ...) { \ + if ((c)) { \ + MSG(__VA_ARGS__); \ + MSG(" \n"); \ + abort(); \ + } \ +} + + +static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize) +{ + const char* const ip1 = (const char*)buff1; + const char* const ip2 = (const char*)buff2; + size_t pos; + + for (pos=0; pos= LZ4_compressBound(srcSize)` + * for compression to be guaranteed to work */ +static void roundTripTest(void* resultBuff, size_t resultBuffCapacity, + void* compressedBuff, size_t compressedBuffCapacity, + const void* srcBuff, size_t srcSize) +{ + const int minCLevel = 1; + const int maxClevel = 12; + const int cLevelSpan = maxClevel - minCLevel; + size_t const hashLength = MIN(16, srcSize); + unsigned const h32 = XXH32(srcBuff, hashLength, 0); + int const randL = h32 % (cLevelSpan+1); + int const cLevel = minCLevel + randL; + int const realCLevel = (cLevel * 0) + 9; /* <=== Currently : only test level 9 */ + int const cSize = LZ4_compress_HC(srcBuff, compressedBuff, srcSize, compressedBuffCapacity, realCLevel); + CONTROL_MSG(cSize == 0, "Compression error !"); + + { int const dSize = LZ4_decompress_safe(compressedBuff, resultBuff, cSize, resultBuffCapacity); + CONTROL_MSG(dSize < 0, "Decompression detected an error !"); + CONTROL_MSG(dSize != (int)srcSize, "Decompression corruption error : wrong decompressed size !"); + } + + /* check potential content corruption error */ + assert(resultBuffCapacity >= srcSize); + { size_t const errorPos = checkBuffers(srcBuff, resultBuff, srcSize); + CONTROL_MSG(errorPos != srcSize, + "Silent decoding corruption, at pos %u !!!", + (unsigned)errorPos); + } + +} + +static void roundTripCheck(const void* srcBuff, size_t srcSize) +{ + size_t const cBuffSize = LZ4_compressBound(srcSize); + void* const cBuff = malloc(cBuffSize); + void* const rBuff = malloc(cBuffSize); + + if (!cBuff || !rBuff) { + fprintf(stderr, "not enough memory ! \n"); + exit(1); + } + + roundTripTest(rBuff, cBuffSize, + cBuff, cBuffSize, + srcBuff, srcSize); + + free(rBuff); + free(cBuff); +} + + +static size_t getFileSize(const char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); + if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ +#else + struct stat statbuf; + r = stat(infilename, &statbuf); + if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ +#endif + return (size_t)statbuf.st_size; +} + + +static int isDirectory(const char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); + if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; +#else + struct stat statbuf; + r = stat(infilename, &statbuf); + if (!r && S_ISDIR(statbuf.st_mode)) return 1; +#endif + return 0; +} + + +/** loadFile() : + * requirement : `buffer` size >= `fileSize` */ +static void loadFile(void* buffer, const char* fileName, size_t fileSize) +{ + FILE* const f = fopen(fileName, "rb"); + if (isDirectory(fileName)) { + fprintf(stderr, "Ignoring %s directory \n", fileName); + exit(2); + } + if (f==NULL) { + fprintf(stderr, "Impossible to open %s \n", fileName); + exit(3); + } + { size_t const readSize = fread(buffer, 1, fileSize, f); + if (readSize != fileSize) { + fprintf(stderr, "Error reading %s \n", fileName); + exit(5); + } } + fclose(f); +} + + +static void fileCheck(const char* fileName) +{ + size_t const fileSize = getFileSize(fileName); + void* const buffer = malloc(fileSize + !fileSize /* avoid 0 */); + if (!buffer) { + fprintf(stderr, "not enough memory \n"); + exit(4); + } + loadFile(buffer, fileName, fileSize); + roundTripCheck(buffer, fileSize); + free (buffer); +} + + +int main(int argCount, const char** argv) +{ + int const argNb = 1; + + if (argCount < 2) { + fprintf(stderr, "Error : no argument : need input file \n"); + exit(9); + } + + fileCheck(argv[argNb]); + fprintf(stderr, "no pb detected \n"); + return 0; +} -- cgit v0.12 From 943fa6244a3985541bc93250d35ea4f4517fb266 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 4 Sep 2018 17:37:56 -0700 Subject: fix minor cast warning for C++ compilation --- tests/.gitignore | 5 ++++- tests/roundTripTest.c | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/.gitignore b/tests/.gitignore index 58947f7..9aa42a0 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,5 +1,5 @@ -# test build artefacts +# build artefacts datagen frametest frametest32 @@ -14,3 +14,6 @@ checkTag # test artefacts tmp* versionsTest + +# local tests +afl diff --git a/tests/roundTripTest.c b/tests/roundTripTest.c index 2f161d8..2caa2bc 100644 --- a/tests/roundTripTest.c +++ b/tests/roundTripTest.c @@ -82,10 +82,10 @@ static void roundTripTest(void* resultBuff, size_t resultBuffCapacity, int const randL = h32 % (cLevelSpan+1); int const cLevel = minCLevel + randL; int const realCLevel = (cLevel * 0) + 9; /* <=== Currently : only test level 9 */ - int const cSize = LZ4_compress_HC(srcBuff, compressedBuff, srcSize, compressedBuffCapacity, realCLevel); + int const cSize = LZ4_compress_HC((const char*)srcBuff, (char*)compressedBuff, (int)srcSize, (int)compressedBuffCapacity, realCLevel); CONTROL_MSG(cSize == 0, "Compression error !"); - { int const dSize = LZ4_decompress_safe(compressedBuff, resultBuff, cSize, resultBuffCapacity); + { int const dSize = LZ4_decompress_safe((const char*)compressedBuff, (char*)resultBuff, cSize, (int)resultBuffCapacity); CONTROL_MSG(dSize < 0, "Decompression detected an error !"); CONTROL_MSG(dSize != (int)srcSize, "Decompression corruption error : wrong decompressed size !"); } @@ -102,7 +102,7 @@ static void roundTripTest(void* resultBuff, size_t resultBuffCapacity, static void roundTripCheck(const void* srcBuff, size_t srcSize) { - size_t const cBuffSize = LZ4_compressBound(srcSize); + size_t const cBuffSize = LZ4_compressBound((int)srcSize); void* const cBuff = malloc(cBuffSize); void* const rBuff = malloc(cBuffSize); -- cgit v0.12 From 2e4847c2d55fb4c4f5dc833e61b5f374da8407e6 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 4 Sep 2018 18:21:40 -0700 Subject: fixed #560 it was a fairly complex scenario, involving source files > 64K and some extraordinary conditions related to specific layout of ranges of zeroes. and only on level 9. --- lib/lz4hc.c | 4 ++++ programs/bench.c | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 8108ea0..236dc5b 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -327,6 +327,8 @@ LZ4HC_InsertAndGetWiderMatch ( if (lookBackLength==0) { /* no back possible */ size_t const maxML = MIN(currentSegmentLength, srcPatternLength); if ((size_t)longest < maxML) { + assert(base + matchIndex < ip); + if (ip - (base+matchIndex) > MAX_DISTANCE) break; assert(maxML < 2 GB); longest = (int)maxML; *matchpos = base + matchIndex; /* virtual pos, relative to ip, to retrieve offset */ @@ -450,6 +452,8 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( *op += length; /* Encode Offset */ + assert(*ip > match); + assert( (*ip - match) <= MAX_DISTANCE ); LZ4_writeLE16(*op, (U16)(*ip-match)); *op += 2; /* Encode MatchLength */ diff --git a/programs/bench.c b/programs/bench.c index 9dc31c4..11bf044 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -329,7 +329,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); if (crcOrig!=crcCheck) { size_t u; - DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck); + DISPLAY("\n!!! WARNING !!! %17s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck); for (u=0; u Date: Tue, 4 Sep 2018 18:23:21 -0700 Subject: made roundTripTest fully general no longer "locked" on level 9 --- tests/roundTripTest.c | 101 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 73 insertions(+), 28 deletions(-) diff --git a/tests/roundTripTest.c b/tests/roundTripTest.c index 2caa2bc..2d34451 100644 --- a/tests/roundTripTest.c +++ b/tests/roundTripTest.c @@ -10,7 +10,7 @@ /* * This program takes a file in input, - * performs an LZ4 round-trip test (compression - decompress) + * performs an LZ4 round-trip test (compress + decompress) * compares the result with original * and generates an abort() on corruption detection, * in order for afl to register the event as a crash. @@ -18,6 +18,15 @@ /*=========================================== +* Tuning Constant +*==========================================*/ +#ifndef MIN_CLEVEL +# define MIN_CLEVEL (int)(-5) +#endif + + + +/*=========================================== * Dependencies *==========================================*/ #include /* size_t */ @@ -62,27 +71,42 @@ static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize return pos; } + +/* select a compression level + * based on first bytes present in a reference buffer */ +static int select_clevel(const void* refBuff, size_t refBuffSize) +{ + const int minCLevel = MIN_CLEVEL; + const int maxClevel = LZ4HC_CLEVEL_MAX; + const int cLevelSpan = maxClevel - minCLevel; + size_t const hashLength = MIN(16, refBuffSize); + unsigned const h32 = XXH32(refBuff, hashLength, 0); + int const randL = h32 % (cLevelSpan+1); + + return minCLevel + randL; +} + + +typedef int (*compressFn)(const char* src, char* dst, int srcSize, int dstSize, int cLevel); + + /** roundTripTest() : * Compresses `srcBuff` into `compressedBuff`, * then decompresses `compressedBuff` into `resultBuff`. - * Compression level used is derived from content's head bytes. - * This function abort() if it detects any round-trip error, - * so if it returns, round trip is considered successfully validated. + * If clevel==0, compression level is derived from srcBuff's content head bytes. + * This function abort() if it detects any round-trip error. + * Therefore, if it returns, round trip is considered successfully validated. * Note : `compressedBuffCapacity` should be `>= LZ4_compressBound(srcSize)` * for compression to be guaranteed to work */ static void roundTripTest(void* resultBuff, size_t resultBuffCapacity, void* compressedBuff, size_t compressedBuffCapacity, - const void* srcBuff, size_t srcSize) + const void* srcBuff, size_t srcSize, + int clevel) { - const int minCLevel = 1; - const int maxClevel = 12; - const int cLevelSpan = maxClevel - minCLevel; - size_t const hashLength = MIN(16, srcSize); - unsigned const h32 = XXH32(srcBuff, hashLength, 0); - int const randL = h32 % (cLevelSpan+1); - int const cLevel = minCLevel + randL; - int const realCLevel = (cLevel * 0) + 9; /* <=== Currently : only test level 9 */ - int const cSize = LZ4_compress_HC((const char*)srcBuff, (char*)compressedBuff, (int)srcSize, (int)compressedBuffCapacity, realCLevel); + int const proposed_clevel = clevel ? clevel : select_clevel(srcBuff, srcSize); + int const selected_clevel = proposed_clevel < 0 ? -proposed_clevel : proposed_clevel; /* if level < 0, it becomes an accelearion value */ + compressFn compress = selected_clevel >= LZ4HC_CLEVEL_MIN ? LZ4_compress_HC : LZ4_compress_fast; + int const cSize = compress((const char*)srcBuff, (char*)compressedBuff, (int)srcSize, (int)compressedBuffCapacity, selected_clevel); CONTROL_MSG(cSize == 0, "Compression error !"); { int const dSize = LZ4_decompress_safe((const char*)compressedBuff, (char*)resultBuff, cSize, (int)resultBuffCapacity); @@ -100,7 +124,7 @@ static void roundTripTest(void* resultBuff, size_t resultBuffCapacity, } -static void roundTripCheck(const void* srcBuff, size_t srcSize) +static void roundTripCheck(const void* srcBuff, size_t srcSize, int clevel) { size_t const cBuffSize = LZ4_compressBound((int)srcSize); void* const cBuff = malloc(cBuffSize); @@ -113,7 +137,8 @@ static void roundTripCheck(const void* srcBuff, size_t srcSize) roundTripTest(rBuff, cBuffSize, cBuff, cBuffSize, - srcBuff, srcSize); + srcBuff, srcSize, + clevel); free(rBuff); free(cBuff); @@ -158,46 +183,66 @@ static void loadFile(void* buffer, const char* fileName, size_t fileSize) { FILE* const f = fopen(fileName, "rb"); if (isDirectory(fileName)) { - fprintf(stderr, "Ignoring %s directory \n", fileName); + MSG("Ignoring %s directory \n", fileName); exit(2); } if (f==NULL) { - fprintf(stderr, "Impossible to open %s \n", fileName); + MSG("Impossible to open %s \n", fileName); exit(3); } { size_t const readSize = fread(buffer, 1, fileSize, f); if (readSize != fileSize) { - fprintf(stderr, "Error reading %s \n", fileName); + MSG("Error reading %s \n", fileName); exit(5); } } fclose(f); } -static void fileCheck(const char* fileName) +static void fileCheck(const char* fileName, int clevel) { size_t const fileSize = getFileSize(fileName); void* const buffer = malloc(fileSize + !fileSize /* avoid 0 */); if (!buffer) { - fprintf(stderr, "not enough memory \n"); + MSG("not enough memory \n"); exit(4); } loadFile(buffer, fileName, fileSize); - roundTripCheck(buffer, fileSize); + roundTripCheck(buffer, fileSize, clevel); free (buffer); } +int bad_usage(const char* exeName) +{ + MSG(" \n"); + MSG("bad usage: \n"); + MSG(" \n"); + MSG("%s [Options] fileName \n", exeName); + MSG(" \n"); + MSG("Options: \n"); + MSG("-# : use #=[0-9] compression level (default:0 == random) \n"); + return 1; +} + + int main(int argCount, const char** argv) { - int const argNb = 1; + const char* const exeName = argv[0]; + int argNb = 1; + int clevel = 0; + + assert(argCount >= 1); + if (argCount < 2) return bad_usage(exeName); - if (argCount < 2) { - fprintf(stderr, "Error : no argument : need input file \n"); - exit(9); + if (argv[1][0] == '-') { + clevel = argv[1][1] - '0'; + argNb = 2; } - fileCheck(argv[argNb]); - fprintf(stderr, "no pb detected \n"); + if (argNb >= argCount) return bad_usage(exeName); + + fileCheck(argv[argNb], clevel); + MSG("no pb detected \n"); return 0; } -- cgit v0.12 From 30f6f34328733ec4e74c78c06f667810db0417df Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 5 Sep 2018 11:25:10 -0700 Subject: removed one assert() condition which is not correct when using LZ4_HC with dictionary and starting from a low address (<0x10000). --- lib/lz4hc.c | 3 +-- tests/fuzzer.c | 15 ++++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 236dc5b..e913ee7 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -452,8 +452,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( *op += length; /* Encode Offset */ - assert(*ip > match); - assert( (*ip - match) <= MAX_DISTANCE ); + assert( (*ip - match) <= MAX_DISTANCE ); /* note : consider providing offset as a value, rather than as a pointer difference */ LZ4_writeLE16(*op, (U16)(*ip-match)); *op += 2; /* Encode MatchLength */ diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 5a92f8f..6c79515 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -323,12 +323,17 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c int result = 0; unsigned cycleNb; -# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \ - printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; } +# define FUZ_CHECKTEST(cond, ...) \ + if (cond) { \ + printf("Test %u : ", testNb); printf(__VA_ARGS__); \ + printf(" (seed %u, cycle %u) \n", seed, cycleNb); \ + goto _output_error; \ + } + # define FUZ_DISPLAYTEST(...) { \ testNb++; \ if (g_displayLevel>=4) { \ - printf("\r%4u - %2u ", cycleNb, testNb); \ + printf("\r%4u - %2u :", cycleNb, testNb); \ printf(" " __VA_ARGS__); \ printf(" "); \ fflush(stdout); \ @@ -805,7 +810,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)"); FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); - FUZ_DISPLAYTEST(); + FUZ_DISPLAYTEST("LZ4_decompress_safe_usingDict with a too small output buffer"); { U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2; if ((U32)blockSize > missingBytes) { decodedBuffer[blockSize-missingBytes] = 0; @@ -815,7 +820,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c } } /* Compress HC using External dictionary */ - FUZ_DISPLAYTEST(); + FUZ_DISPLAYTEST("LZ4_compress_HC_continue with an external dictionary"); dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */ if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; LZ4_resetStreamHC (&LZ4dictHC, compressionLevel); -- cgit v0.12