diff options
| -rw-r--r-- | lib/lz4hc.c | 3 | ||||
| -rw-r--r-- | programs/bench.c | 2 | ||||
| -rw-r--r-- | tests/.gitignore | 6 | ||||
| -rw-r--r-- | tests/Makefile | 5 | ||||
| -rw-r--r-- | tests/fuzzer.c | 15 | ||||
| -rw-r--r-- | tests/roundTripTest.c | 248 |
6 files changed, 271 insertions, 8 deletions
diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 8108ea0..e913ee7 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -327,6 +327,8 @@ LZ4HC_InsertAndGetWiderMatch ( if (lookBackLength==0) { /* no back possible */ size_t const maxML = MIN(currentSegmentLength, srcPatternLength); if ((size_t)longest < maxML) { + assert(base + matchIndex < ip); + if (ip - (base+matchIndex) > MAX_DISTANCE) break; assert(maxML < 2 GB); longest = (int)maxML; *matchpos = base + matchIndex; /* virtual pos, relative to ip, to retrieve offset */ @@ -450,6 +452,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( *op += length; /* Encode Offset */ + assert( (*ip - match) <= MAX_DISTANCE ); /* note : consider providing offset as a value, rather than as a pointer difference */ LZ4_writeLE16(*op, (U16)(*ip-match)); *op += 2; /* Encode MatchLength */ diff --git a/programs/bench.c b/programs/bench.c index 9dc31c4..11bf044 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -329,7 +329,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); if (crcOrig!=crcCheck) { size_t u; - DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck); + DISPLAY("\n!!! WARNING !!! %17s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck); for (u=0; u<srcSize; u++) { if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u]) { U32 segNb, bNb, pos; diff --git a/tests/.gitignore b/tests/.gitignore index 36dff42..9aa42a0 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,5 +1,5 @@ -# test build artefacts +# build artefacts datagen frametest frametest32 @@ -8,8 +8,12 @@ fullbench32 fuzzer fuzzer32 fasttest +roundTripTest checkTag # test artefacts tmp* versionsTest + +# local tests +afl diff --git a/tests/Makefile b/tests/Makefile index 81033b5..bc43234 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -63,7 +63,7 @@ NB_LOOPS ?= -i1 default: all -all: fullbench fuzzer frametest datagen +all: fullbench fuzzer frametest roundTripTest datagen all32: CFLAGS+=-m32 all32: all @@ -103,6 +103,9 @@ fuzzer : lz4.o lz4hc.o xxhash.o fuzzer.c frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c $(CC) $(FLAGS) $^ -o $@$(EXT) +roundTripTest : lz4.o lz4hc.o xxhash.o roundTripTest.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + datagen : $(PRGDIR)/datagen.c datagencli.c $(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 5a92f8f..6c79515 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -323,12 +323,17 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c int result = 0; unsigned cycleNb; -# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \ - printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; } +# define FUZ_CHECKTEST(cond, ...) \ + if (cond) { \ + printf("Test %u : ", testNb); printf(__VA_ARGS__); \ + printf(" (seed %u, cycle %u) \n", seed, cycleNb); \ + goto _output_error; \ + } + # define FUZ_DISPLAYTEST(...) { \ testNb++; \ if (g_displayLevel>=4) { \ - printf("\r%4u - %2u ", cycleNb, testNb); \ + printf("\r%4u - %2u :", cycleNb, testNb); \ printf(" " __VA_ARGS__); \ printf(" "); \ fflush(stdout); \ @@ -805,7 +810,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)"); FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); - FUZ_DISPLAYTEST(); + FUZ_DISPLAYTEST("LZ4_decompress_safe_usingDict with a too small output buffer"); { U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2; if ((U32)blockSize > missingBytes) { decodedBuffer[blockSize-missingBytes] = 0; @@ -815,7 +820,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c } } /* Compress HC using External dictionary */ - FUZ_DISPLAYTEST(); + FUZ_DISPLAYTEST("LZ4_compress_HC_continue with an external dictionary"); dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */ if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; LZ4_resetStreamHC (&LZ4dictHC, compressionLevel); diff --git a/tests/roundTripTest.c b/tests/roundTripTest.c new file mode 100644 index 0000000..2d34451 --- /dev/null +++ b/tests/roundTripTest.c @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* + * This program takes a file in input, + * performs an LZ4 round-trip test (compress + decompress) + * compares the result with original + * and generates an abort() on corruption detection, + * in order for afl to register the event as a crash. +*/ + + +/*=========================================== +* Tuning Constant +*==========================================*/ +#ifndef MIN_CLEVEL +# define MIN_CLEVEL (int)(-5) +#endif + + + +/*=========================================== +* Dependencies +*==========================================*/ +#include <stddef.h> /* size_t */ +#include <stdlib.h> /* malloc, free, exit */ +#include <stdio.h> /* fprintf */ +#include <string.h> /* strcmp */ +#include <assert.h> +#include <sys/types.h> /* stat */ +#include <sys/stat.h> /* stat */ +#include "xxhash.h" + +#include "lz4.h" +#include "lz4hc.h" + + +/*=========================================== +* Macros +*==========================================*/ +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + +#define MSG(...) fprintf(stderr, __VA_ARGS__) + +#define CONTROL_MSG(c, ...) { \ + if ((c)) { \ + MSG(__VA_ARGS__); \ + MSG(" \n"); \ + abort(); \ + } \ +} + + +static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize) +{ + const char* const ip1 = (const char*)buff1; + const char* const ip2 = (const char*)buff2; + size_t pos; + + for (pos=0; pos<buffSize; pos++) + if (ip1[pos]!=ip2[pos]) + break; + + return pos; +} + + +/* select a compression level + * based on first bytes present in a reference buffer */ +static int select_clevel(const void* refBuff, size_t refBuffSize) +{ + const int minCLevel = MIN_CLEVEL; + const int maxClevel = LZ4HC_CLEVEL_MAX; + const int cLevelSpan = maxClevel - minCLevel; + size_t const hashLength = MIN(16, refBuffSize); + unsigned const h32 = XXH32(refBuff, hashLength, 0); + int const randL = h32 % (cLevelSpan+1); + + return minCLevel + randL; +} + + +typedef int (*compressFn)(const char* src, char* dst, int srcSize, int dstSize, int cLevel); + + +/** roundTripTest() : + * Compresses `srcBuff` into `compressedBuff`, + * then decompresses `compressedBuff` into `resultBuff`. + * If clevel==0, compression level is derived from srcBuff's content head bytes. + * This function abort() if it detects any round-trip error. + * Therefore, if it returns, round trip is considered successfully validated. + * Note : `compressedBuffCapacity` should be `>= LZ4_compressBound(srcSize)` + * for compression to be guaranteed to work */ +static void roundTripTest(void* resultBuff, size_t resultBuffCapacity, + void* compressedBuff, size_t compressedBuffCapacity, + const void* srcBuff, size_t srcSize, + int clevel) +{ + int const proposed_clevel = clevel ? clevel : select_clevel(srcBuff, srcSize); + int const selected_clevel = proposed_clevel < 0 ? -proposed_clevel : proposed_clevel; /* if level < 0, it becomes an accelearion value */ + compressFn compress = selected_clevel >= LZ4HC_CLEVEL_MIN ? LZ4_compress_HC : LZ4_compress_fast; + int const cSize = compress((const char*)srcBuff, (char*)compressedBuff, (int)srcSize, (int)compressedBuffCapacity, selected_clevel); + CONTROL_MSG(cSize == 0, "Compression error !"); + + { int const dSize = LZ4_decompress_safe((const char*)compressedBuff, (char*)resultBuff, cSize, (int)resultBuffCapacity); + CONTROL_MSG(dSize < 0, "Decompression detected an error !"); + CONTROL_MSG(dSize != (int)srcSize, "Decompression corruption error : wrong decompressed size !"); + } + + /* check potential content corruption error */ + assert(resultBuffCapacity >= srcSize); + { size_t const errorPos = checkBuffers(srcBuff, resultBuff, srcSize); + CONTROL_MSG(errorPos != srcSize, + "Silent decoding corruption, at pos %u !!!", + (unsigned)errorPos); + } + +} + +static void roundTripCheck(const void* srcBuff, size_t srcSize, int clevel) +{ + size_t const cBuffSize = LZ4_compressBound((int)srcSize); + void* const cBuff = malloc(cBuffSize); + void* const rBuff = malloc(cBuffSize); + + if (!cBuff || !rBuff) { + fprintf(stderr, "not enough memory ! \n"); + exit(1); + } + + roundTripTest(rBuff, cBuffSize, + cBuff, cBuffSize, + srcBuff, srcSize, + clevel); + + free(rBuff); + free(cBuff); +} + + +static size_t getFileSize(const char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); + if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ +#else + struct stat statbuf; + r = stat(infilename, &statbuf); + if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ +#endif + return (size_t)statbuf.st_size; +} + + +static int isDirectory(const char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); + if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; +#else + struct stat statbuf; + r = stat(infilename, &statbuf); + if (!r && S_ISDIR(statbuf.st_mode)) return 1; +#endif + return 0; +} + + +/** loadFile() : + * requirement : `buffer` size >= `fileSize` */ +static void loadFile(void* buffer, const char* fileName, size_t fileSize) +{ + FILE* const f = fopen(fileName, "rb"); + if (isDirectory(fileName)) { + MSG("Ignoring %s directory \n", fileName); + exit(2); + } + if (f==NULL) { + MSG("Impossible to open %s \n", fileName); + exit(3); + } + { size_t const readSize = fread(buffer, 1, fileSize, f); + if (readSize != fileSize) { + MSG("Error reading %s \n", fileName); + exit(5); + } } + fclose(f); +} + + +static void fileCheck(const char* fileName, int clevel) +{ + size_t const fileSize = getFileSize(fileName); + void* const buffer = malloc(fileSize + !fileSize /* avoid 0 */); + if (!buffer) { + MSG("not enough memory \n"); + exit(4); + } + loadFile(buffer, fileName, fileSize); + roundTripCheck(buffer, fileSize, clevel); + free (buffer); +} + + +int bad_usage(const char* exeName) +{ + MSG(" \n"); + MSG("bad usage: \n"); + MSG(" \n"); + MSG("%s [Options] fileName \n", exeName); + MSG(" \n"); + MSG("Options: \n"); + MSG("-# : use #=[0-9] compression level (default:0 == random) \n"); + return 1; +} + + +int main(int argCount, const char** argv) +{ + const char* const exeName = argv[0]; + int argNb = 1; + int clevel = 0; + + assert(argCount >= 1); + if (argCount < 2) return bad_usage(exeName); + + if (argv[1][0] == '-') { + clevel = argv[1][1] - '0'; + argNb = 2; + } + + if (argNb >= argCount) return bad_usage(exeName); + + fileCheck(argv[argNb], clevel); + MSG("no pb detected \n"); + return 0; +} |