summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--lib/lz4hc.c3
-rw-r--r--programs/bench.c2
-rw-r--r--tests/.gitignore6
-rw-r--r--tests/Makefile5
-rw-r--r--tests/fuzzer.c15
-rw-r--r--tests/roundTripTest.c248
6 files changed, 271 insertions, 8 deletions
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 8108ea0..e913ee7 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -327,6 +327,8 @@ LZ4HC_InsertAndGetWiderMatch (
if (lookBackLength==0) { /* no back possible */
size_t const maxML = MIN(currentSegmentLength, srcPatternLength);
if ((size_t)longest < maxML) {
+ assert(base + matchIndex < ip);
+ if (ip - (base+matchIndex) > MAX_DISTANCE) break;
assert(maxML < 2 GB);
longest = (int)maxML;
*matchpos = base + matchIndex; /* virtual pos, relative to ip, to retrieve offset */
@@ -450,6 +452,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
*op += length;
/* Encode Offset */
+ assert( (*ip - match) <= MAX_DISTANCE ); /* note : consider providing offset as a value, rather than as a pointer difference */
LZ4_writeLE16(*op, (U16)(*ip-match)); *op += 2;
/* Encode MatchLength */
diff --git a/programs/bench.c b/programs/bench.c
index 9dc31c4..11bf044 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -329,7 +329,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
{ U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
if (crcOrig!=crcCheck) {
size_t u;
- DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
+ DISPLAY("\n!!! WARNING !!! %17s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
for (u=0; u<srcSize; u++) {
if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u]) {
U32 segNb, bNb, pos;
diff --git a/tests/.gitignore b/tests/.gitignore
index 36dff42..9aa42a0 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -1,5 +1,5 @@
-# test build artefacts
+# build artefacts
datagen
frametest
frametest32
@@ -8,8 +8,12 @@ fullbench32
fuzzer
fuzzer32
fasttest
+roundTripTest
checkTag
# test artefacts
tmp*
versionsTest
+
+# local tests
+afl
diff --git a/tests/Makefile b/tests/Makefile
index 81033b5..bc43234 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -63,7 +63,7 @@ NB_LOOPS ?= -i1
default: all
-all: fullbench fuzzer frametest datagen
+all: fullbench fuzzer frametest roundTripTest datagen
all32: CFLAGS+=-m32
all32: all
@@ -103,6 +103,9 @@ fuzzer : lz4.o lz4hc.o xxhash.o fuzzer.c
frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
+roundTripTest : lz4.o lz4hc.o xxhash.o roundTripTest.c
+ $(CC) $(FLAGS) $^ -o $@$(EXT)
+
datagen : $(PRGDIR)/datagen.c datagencli.c
$(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT)
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 5a92f8f..6c79515 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -323,12 +323,17 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
int result = 0;
unsigned cycleNb;
-# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \
- printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; }
+# define FUZ_CHECKTEST(cond, ...) \
+ if (cond) { \
+ printf("Test %u : ", testNb); printf(__VA_ARGS__); \
+ printf(" (seed %u, cycle %u) \n", seed, cycleNb); \
+ goto _output_error; \
+ }
+
# define FUZ_DISPLAYTEST(...) { \
testNb++; \
if (g_displayLevel>=4) { \
- printf("\r%4u - %2u ", cycleNb, testNb); \
+ printf("\r%4u - %2u :", cycleNb, testNb); \
printf(" " __VA_ARGS__); \
printf(" "); \
fflush(stdout); \
@@ -805,7 +810,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
- FUZ_DISPLAYTEST();
+ FUZ_DISPLAYTEST("LZ4_decompress_safe_usingDict with a too small output buffer");
{ U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
if ((U32)blockSize > missingBytes) {
decodedBuffer[blockSize-missingBytes] = 0;
@@ -815,7 +820,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
} }
/* Compress HC using External dictionary */
- FUZ_DISPLAYTEST();
+ FUZ_DISPLAYTEST("LZ4_compress_HC_continue with an external dictionary");
dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
LZ4_resetStreamHC (&LZ4dictHC, compressionLevel);
diff --git a/tests/roundTripTest.c b/tests/roundTripTest.c
new file mode 100644
index 0000000..2d34451
--- /dev/null
+++ b/tests/roundTripTest.c
@@ -0,0 +1,248 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/*
+ * This program takes a file in input,
+ * performs an LZ4 round-trip test (compress + decompress)
+ * compares the result with original
+ * and generates an abort() on corruption detection,
+ * in order for afl to register the event as a crash.
+*/
+
+
+/*===========================================
+* Tuning Constant
+*==========================================*/
+#ifndef MIN_CLEVEL
+# define MIN_CLEVEL (int)(-5)
+#endif
+
+
+
+/*===========================================
+* Dependencies
+*==========================================*/
+#include <stddef.h> /* size_t */
+#include <stdlib.h> /* malloc, free, exit */
+#include <stdio.h> /* fprintf */
+#include <string.h> /* strcmp */
+#include <assert.h>
+#include <sys/types.h> /* stat */
+#include <sys/stat.h> /* stat */
+#include "xxhash.h"
+
+#include "lz4.h"
+#include "lz4hc.h"
+
+
+/*===========================================
+* Macros
+*==========================================*/
+#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
+
+#define MSG(...) fprintf(stderr, __VA_ARGS__)
+
+#define CONTROL_MSG(c, ...) { \
+ if ((c)) { \
+ MSG(__VA_ARGS__); \
+ MSG(" \n"); \
+ abort(); \
+ } \
+}
+
+
+static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize)
+{
+ const char* const ip1 = (const char*)buff1;
+ const char* const ip2 = (const char*)buff2;
+ size_t pos;
+
+ for (pos=0; pos<buffSize; pos++)
+ if (ip1[pos]!=ip2[pos])
+ break;
+
+ return pos;
+}
+
+
+/* select a compression level
+ * based on first bytes present in a reference buffer */
+static int select_clevel(const void* refBuff, size_t refBuffSize)
+{
+ const int minCLevel = MIN_CLEVEL;
+ const int maxClevel = LZ4HC_CLEVEL_MAX;
+ const int cLevelSpan = maxClevel - minCLevel;
+ size_t const hashLength = MIN(16, refBuffSize);
+ unsigned const h32 = XXH32(refBuff, hashLength, 0);
+ int const randL = h32 % (cLevelSpan+1);
+
+ return minCLevel + randL;
+}
+
+
+typedef int (*compressFn)(const char* src, char* dst, int srcSize, int dstSize, int cLevel);
+
+
+/** roundTripTest() :
+ * Compresses `srcBuff` into `compressedBuff`,
+ * then decompresses `compressedBuff` into `resultBuff`.
+ * If clevel==0, compression level is derived from srcBuff's content head bytes.
+ * This function abort() if it detects any round-trip error.
+ * Therefore, if it returns, round trip is considered successfully validated.
+ * Note : `compressedBuffCapacity` should be `>= LZ4_compressBound(srcSize)`
+ * for compression to be guaranteed to work */
+static void roundTripTest(void* resultBuff, size_t resultBuffCapacity,
+ void* compressedBuff, size_t compressedBuffCapacity,
+ const void* srcBuff, size_t srcSize,
+ int clevel)
+{
+ int const proposed_clevel = clevel ? clevel : select_clevel(srcBuff, srcSize);
+ int const selected_clevel = proposed_clevel < 0 ? -proposed_clevel : proposed_clevel; /* if level < 0, it becomes an accelearion value */
+ compressFn compress = selected_clevel >= LZ4HC_CLEVEL_MIN ? LZ4_compress_HC : LZ4_compress_fast;
+ int const cSize = compress((const char*)srcBuff, (char*)compressedBuff, (int)srcSize, (int)compressedBuffCapacity, selected_clevel);
+ CONTROL_MSG(cSize == 0, "Compression error !");
+
+ { int const dSize = LZ4_decompress_safe((const char*)compressedBuff, (char*)resultBuff, cSize, (int)resultBuffCapacity);
+ CONTROL_MSG(dSize < 0, "Decompression detected an error !");
+ CONTROL_MSG(dSize != (int)srcSize, "Decompression corruption error : wrong decompressed size !");
+ }
+
+ /* check potential content corruption error */
+ assert(resultBuffCapacity >= srcSize);
+ { size_t const errorPos = checkBuffers(srcBuff, resultBuff, srcSize);
+ CONTROL_MSG(errorPos != srcSize,
+ "Silent decoding corruption, at pos %u !!!",
+ (unsigned)errorPos);
+ }
+
+}
+
+static void roundTripCheck(const void* srcBuff, size_t srcSize, int clevel)
+{
+ size_t const cBuffSize = LZ4_compressBound((int)srcSize);
+ void* const cBuff = malloc(cBuffSize);
+ void* const rBuff = malloc(cBuffSize);
+
+ if (!cBuff || !rBuff) {
+ fprintf(stderr, "not enough memory ! \n");
+ exit(1);
+ }
+
+ roundTripTest(rBuff, cBuffSize,
+ cBuff, cBuffSize,
+ srcBuff, srcSize,
+ clevel);
+
+ free(rBuff);
+ free(cBuff);
+}
+
+
+static size_t getFileSize(const char* infilename)
+{
+ int r;
+#if defined(_MSC_VER)
+ struct _stat64 statbuf;
+ r = _stat64(infilename, &statbuf);
+ if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */
+#else
+ struct stat statbuf;
+ r = stat(infilename, &statbuf);
+ if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
+#endif
+ return (size_t)statbuf.st_size;
+}
+
+
+static int isDirectory(const char* infilename)
+{
+ int r;
+#if defined(_MSC_VER)
+ struct _stat64 statbuf;
+ r = _stat64(infilename, &statbuf);
+ if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
+#else
+ struct stat statbuf;
+ r = stat(infilename, &statbuf);
+ if (!r && S_ISDIR(statbuf.st_mode)) return 1;
+#endif
+ return 0;
+}
+
+
+/** loadFile() :
+ * requirement : `buffer` size >= `fileSize` */
+static void loadFile(void* buffer, const char* fileName, size_t fileSize)
+{
+ FILE* const f = fopen(fileName, "rb");
+ if (isDirectory(fileName)) {
+ MSG("Ignoring %s directory \n", fileName);
+ exit(2);
+ }
+ if (f==NULL) {
+ MSG("Impossible to open %s \n", fileName);
+ exit(3);
+ }
+ { size_t const readSize = fread(buffer, 1, fileSize, f);
+ if (readSize != fileSize) {
+ MSG("Error reading %s \n", fileName);
+ exit(5);
+ } }
+ fclose(f);
+}
+
+
+static void fileCheck(const char* fileName, int clevel)
+{
+ size_t const fileSize = getFileSize(fileName);
+ void* const buffer = malloc(fileSize + !fileSize /* avoid 0 */);
+ if (!buffer) {
+ MSG("not enough memory \n");
+ exit(4);
+ }
+ loadFile(buffer, fileName, fileSize);
+ roundTripCheck(buffer, fileSize, clevel);
+ free (buffer);
+}
+
+
+int bad_usage(const char* exeName)
+{
+ MSG(" \n");
+ MSG("bad usage: \n");
+ MSG(" \n");
+ MSG("%s [Options] fileName \n", exeName);
+ MSG(" \n");
+ MSG("Options: \n");
+ MSG("-# : use #=[0-9] compression level (default:0 == random) \n");
+ return 1;
+}
+
+
+int main(int argCount, const char** argv)
+{
+ const char* const exeName = argv[0];
+ int argNb = 1;
+ int clevel = 0;
+
+ assert(argCount >= 1);
+ if (argCount < 2) return bad_usage(exeName);
+
+ if (argv[1][0] == '-') {
+ clevel = argv[1][1] - '0';
+ argNb = 2;
+ }
+
+ if (argNb >= argCount) return bad_usage(exeName);
+
+ fileCheck(argv[argNb], clevel);
+ MSG("no pb detected \n");
+ return 0;
+}