diff options
| -rw-r--r-- | examples/frameCompress.c | 2 | ||||
| -rw-r--r-- | lib/lz4hc.c | 3 | ||||
| -rw-r--r-- | programs/Makefile | 24 | ||||
| -rw-r--r-- | programs/bench.c | 2 | ||||
| -rw-r--r-- | programs/lz4io.c | 31 | ||||
| -rw-r--r-- | tests/Makefile | 5 | ||||
| -rw-r--r-- | tests/fuzzer.c | 15 | ||||
| -rw-r--r-- | tests/roundTripTest.c | 248 |
8 files changed, 300 insertions, 30 deletions
diff --git a/examples/frameCompress.c b/examples/frameCompress.c index 9bfea48..0f8910d 100644 --- a/examples/frameCompress.c +++ b/examples/frameCompress.c @@ -260,7 +260,7 @@ decompress_file_allocDst(FILE* f_in, FILE* f_out, int const decompressionResult = decompress_file_internal( f_in, f_out, dctx, - src, srcCapacity, readSize, consumedSize, + src, srcCapacity, readSize-consumedSize, consumedSize, dst, dstCapacity); free(dst); diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 8108ea0..e913ee7 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -327,6 +327,8 @@ LZ4HC_InsertAndGetWiderMatch ( if (lookBackLength==0) { /* no back possible */ size_t const maxML = MIN(currentSegmentLength, srcPatternLength); if ((size_t)longest < maxML) { + assert(base + matchIndex < ip); + if (ip - (base+matchIndex) > MAX_DISTANCE) break; assert(maxML < 2 GB); longest = (int)maxML; *matchpos = base + matchIndex; /* virtual pos, relative to ip, to retrieve offset */ @@ -450,6 +452,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( *op += length; /* Encode Offset */ + assert( (*ip - match) <= MAX_DISTANCE ); /* note : consider providing offset as a value, rather than as a pointer difference */ LZ4_writeLE16(*op, (U16)(*ip-match)); *op += 2; /* Encode MatchLength */ diff --git a/programs/Makefile b/programs/Makefile index 655efbc..98366ad 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -82,7 +82,7 @@ lz4-release: DEBUGFLAGS= lz4-release: lz4 lz4c: lz4 - ln -s lz4 lz4c + ln -s lz4$(EXT) lz4c$(EXT) lz4c32: CFLAGS += -m32 lz4c32 : $(SRCFILES) @@ -102,7 +102,7 @@ preview-man: clean-man man clean: @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) @$(RM) core *.o *.test tmp* \ - lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4 lz4cat + lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4$(EXT) lz4cat$(EXT) @echo Cleaning completed @@ -112,10 +112,10 @@ clean: ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku)) unlz4: lz4 - ln -s lz4 unlz4 + ln -s lz4$(EXT) unlz4$(EXT) lz4cat: lz4 - ln -s lz4 lz4cat + ln -s lz4$(EXT) lz4cat$(EXT) DESTDIR ?= # directory variables : GNU conventions prefer lowercase @@ -147,10 +147,10 @@ INSTALL_DATA ?= $(INSTALL) -m 644 install: lz4 @echo Installing binaries @$(INSTALL) -d -m 755 $(DESTDIR)$(bindir)/ $(DESTDIR)$(man1dir)/ - @$(INSTALL_PROGRAM) lz4 $(DESTDIR)$(bindir)/lz4 - @ln -sf lz4 $(DESTDIR)$(bindir)/lz4c - @ln -sf lz4 $(DESTDIR)$(bindir)/lz4cat - @ln -sf lz4 $(DESTDIR)$(bindir)/unlz4 + @$(INSTALL_PROGRAM) lz4$(EXT) $(DESTDIR)$(bindir)/lz4$(EXT) + @ln -sf lz4$(EXT) $(DESTDIR)$(bindir)/lz4c$(EXT) + @ln -sf lz4$(EXT) $(DESTDIR)$(bindir)/lz4cat$(EXT) + @ln -sf lz4$(EXT) $(DESTDIR)$(bindir)/unlz4$(EXT) @echo Installing man pages @$(INSTALL_DATA) lz4.1 $(DESTDIR)$(man1dir)/lz4.1 @ln -sf lz4.1 $(DESTDIR)$(man1dir)/lz4c.1 @@ -159,10 +159,10 @@ install: lz4 @echo lz4 installation completed uninstall: - @$(RM) $(DESTDIR)$(bindir)/lz4cat - @$(RM) $(DESTDIR)$(bindir)/unlz4 - @$(RM) $(DESTDIR)$(bindir)/lz4 - @$(RM) $(DESTDIR)$(bindir)/lz4c + @$(RM) $(DESTDIR)$(bindir)/lz4cat$(EXT) + @$(RM) $(DESTDIR)$(bindir)/unlz4$(EXT) + @$(RM) $(DESTDIR)$(bindir)/lz4$(EXT) + @$(RM) $(DESTDIR)$(bindir)/lz4c$(EXT) @$(RM) $(DESTDIR)$(man1dir)/lz4.1 @$(RM) $(DESTDIR)$(man1dir)/lz4c.1 @$(RM) $(DESTDIR)$(man1dir)/lz4cat.1 diff --git a/programs/bench.c b/programs/bench.c index 9dc31c4..11bf044 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -329,7 +329,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); if (crcOrig!=crcCheck) { size_t u; - DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck); + DISPLAY("\n!!! WARNING !!! %17s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck); for (u=0; u<srcSize; u++) { if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u]) { U32 segNb, bNb, pos; diff --git a/programs/lz4io.c b/programs/lz4io.c index b52c1f3..28d6537 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -628,16 +628,23 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, /* Copy owner, file permissions and modification time */ { stat_t statbuf; - if (strcmp (srcFileName, stdinmark) && strcmp (dstFileName, stdoutmark) && strcmp (dstFileName, nulmark) && UTIL_getFileStat(srcFileName, &statbuf)) + if (strcmp (srcFileName, stdinmark) + && strcmp (dstFileName, stdoutmark) + && strcmp (dstFileName, nulmark) + && UTIL_getFileStat(srcFileName, &statbuf)) { UTIL_setFileStat(dstFileName, &statbuf); - } + } } - if (g_removeSrcFile) { if (remove(srcFileName)) EXM_THROW(40, "Remove error : %s: %s", srcFileName, strerror(errno)); } /* remove source file : --rm */ + if (g_removeSrcFile) { /* remove source file : --rm */ + if (remove(srcFileName)) + EXM_THROW(40, "Remove error : %s: %s", srcFileName, strerror(errno)); + } /* Final Status */ DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", - filesize, compressedfilesize, (double)compressedfilesize/(filesize + !filesize)*100); /* avoid division by zero */ + filesize, compressedfilesize, + (double)compressedfilesize / (filesize + !filesize /* avoid division by zero */ ) * 100); return 0; } @@ -645,21 +652,25 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, int LZ4IO_compressFilename(const char* srcFileName, const char* dstFileName, int compressionLevel) { - clock_t const start = clock(); + UTIL_time_t const timeStart = UTIL_getTime(); + clock_t const cpuStart = clock(); cRess_t const ress = LZ4IO_createCResources(); - int const issueWithSrcFile = LZ4IO_compressFilename_extRess(ress, srcFileName, dstFileName, compressionLevel); + int const result = LZ4IO_compressFilename_extRess(ress, srcFileName, dstFileName, compressionLevel); /* Free resources */ LZ4IO_freeCResources(ress); /* Final Status */ - { clock_t const end = clock(); - double const seconds = (double)(end - start) / CLOCKS_PER_SEC; - DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds); + { clock_t const cpuEnd = clock(); + double const cpuLoad_s = (double)(cpuEnd - cpuStart) / CLOCKS_PER_SEC; + U64 const timeLength_ns = UTIL_clockSpanNano(timeStart); + double const timeLength_s = (double)timeLength_ns / 1000000000; + DISPLAYLEVEL(4, "Completed in %.2f sec (cpu load : %.0f%%)\n", + timeLength_s, (cpuLoad_s / timeLength_s) * 100); } - return issueWithSrcFile; + return result; } diff --git a/tests/Makefile b/tests/Makefile index f270c46..16f2867 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -63,7 +63,7 @@ NB_LOOPS ?= -i1 default: all -all: fullbench fuzzer frametest datagen +all: fullbench fuzzer frametest roundTripTest datagen all32: CFLAGS+=-m32 all32: all @@ -103,6 +103,9 @@ fuzzer : lz4.o lz4hc.o xxhash.o fuzzer.c frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c $(CC) $(FLAGS) $^ -o $@$(EXT) +roundTripTest : lz4.o lz4hc.o xxhash.o roundTripTest.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + datagen : $(PRGDIR)/datagen.c datagencli.c $(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 5a92f8f..6c79515 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -323,12 +323,17 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c int result = 0; unsigned cycleNb; -# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \ - printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; } +# define FUZ_CHECKTEST(cond, ...) \ + if (cond) { \ + printf("Test %u : ", testNb); printf(__VA_ARGS__); \ + printf(" (seed %u, cycle %u) \n", seed, cycleNb); \ + goto _output_error; \ + } + # define FUZ_DISPLAYTEST(...) { \ testNb++; \ if (g_displayLevel>=4) { \ - printf("\r%4u - %2u ", cycleNb, testNb); \ + printf("\r%4u - %2u :", cycleNb, testNb); \ printf(" " __VA_ARGS__); \ printf(" "); \ fflush(stdout); \ @@ -805,7 +810,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)"); FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); - FUZ_DISPLAYTEST(); + FUZ_DISPLAYTEST("LZ4_decompress_safe_usingDict with a too small output buffer"); { U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2; if ((U32)blockSize > missingBytes) { decodedBuffer[blockSize-missingBytes] = 0; @@ -815,7 +820,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c } } /* Compress HC using External dictionary */ - FUZ_DISPLAYTEST(); + FUZ_DISPLAYTEST("LZ4_compress_HC_continue with an external dictionary"); dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */ if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; LZ4_resetStreamHC (&LZ4dictHC, compressionLevel); diff --git a/tests/roundTripTest.c b/tests/roundTripTest.c new file mode 100644 index 0000000..2d34451 --- /dev/null +++ b/tests/roundTripTest.c @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* + * This program takes a file in input, + * performs an LZ4 round-trip test (compress + decompress) + * compares the result with original + * and generates an abort() on corruption detection, + * in order for afl to register the event as a crash. +*/ + + +/*=========================================== +* Tuning Constant +*==========================================*/ +#ifndef MIN_CLEVEL +# define MIN_CLEVEL (int)(-5) +#endif + + + +/*=========================================== +* Dependencies +*==========================================*/ +#include <stddef.h> /* size_t */ +#include <stdlib.h> /* malloc, free, exit */ +#include <stdio.h> /* fprintf */ +#include <string.h> /* strcmp */ +#include <assert.h> +#include <sys/types.h> /* stat */ +#include <sys/stat.h> /* stat */ +#include "xxhash.h" + +#include "lz4.h" +#include "lz4hc.h" + + +/*=========================================== +* Macros +*==========================================*/ +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + +#define MSG(...) fprintf(stderr, __VA_ARGS__) + +#define CONTROL_MSG(c, ...) { \ + if ((c)) { \ + MSG(__VA_ARGS__); \ + MSG(" \n"); \ + abort(); \ + } \ +} + + +static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize) +{ + const char* const ip1 = (const char*)buff1; + const char* const ip2 = (const char*)buff2; + size_t pos; + + for (pos=0; pos<buffSize; pos++) + if (ip1[pos]!=ip2[pos]) + break; + + return pos; +} + + +/* select a compression level + * based on first bytes present in a reference buffer */ +static int select_clevel(const void* refBuff, size_t refBuffSize) +{ + const int minCLevel = MIN_CLEVEL; + const int maxClevel = LZ4HC_CLEVEL_MAX; + const int cLevelSpan = maxClevel - minCLevel; + size_t const hashLength = MIN(16, refBuffSize); + unsigned const h32 = XXH32(refBuff, hashLength, 0); + int const randL = h32 % (cLevelSpan+1); + + return minCLevel + randL; +} + + +typedef int (*compressFn)(const char* src, char* dst, int srcSize, int dstSize, int cLevel); + + +/** roundTripTest() : + * Compresses `srcBuff` into `compressedBuff`, + * then decompresses `compressedBuff` into `resultBuff`. + * If clevel==0, compression level is derived from srcBuff's content head bytes. + * This function abort() if it detects any round-trip error. + * Therefore, if it returns, round trip is considered successfully validated. + * Note : `compressedBuffCapacity` should be `>= LZ4_compressBound(srcSize)` + * for compression to be guaranteed to work */ +static void roundTripTest(void* resultBuff, size_t resultBuffCapacity, + void* compressedBuff, size_t compressedBuffCapacity, + const void* srcBuff, size_t srcSize, + int clevel) +{ + int const proposed_clevel = clevel ? clevel : select_clevel(srcBuff, srcSize); + int const selected_clevel = proposed_clevel < 0 ? -proposed_clevel : proposed_clevel; /* if level < 0, it becomes an accelearion value */ + compressFn compress = selected_clevel >= LZ4HC_CLEVEL_MIN ? LZ4_compress_HC : LZ4_compress_fast; + int const cSize = compress((const char*)srcBuff, (char*)compressedBuff, (int)srcSize, (int)compressedBuffCapacity, selected_clevel); + CONTROL_MSG(cSize == 0, "Compression error !"); + + { int const dSize = LZ4_decompress_safe((const char*)compressedBuff, (char*)resultBuff, cSize, (int)resultBuffCapacity); + CONTROL_MSG(dSize < 0, "Decompression detected an error !"); + CONTROL_MSG(dSize != (int)srcSize, "Decompression corruption error : wrong decompressed size !"); + } + + /* check potential content corruption error */ + assert(resultBuffCapacity >= srcSize); + { size_t const errorPos = checkBuffers(srcBuff, resultBuff, srcSize); + CONTROL_MSG(errorPos != srcSize, + "Silent decoding corruption, at pos %u !!!", + (unsigned)errorPos); + } + +} + +static void roundTripCheck(const void* srcBuff, size_t srcSize, int clevel) +{ + size_t const cBuffSize = LZ4_compressBound((int)srcSize); + void* const cBuff = malloc(cBuffSize); + void* const rBuff = malloc(cBuffSize); + + if (!cBuff || !rBuff) { + fprintf(stderr, "not enough memory ! \n"); + exit(1); + } + + roundTripTest(rBuff, cBuffSize, + cBuff, cBuffSize, + srcBuff, srcSize, + clevel); + + free(rBuff); + free(cBuff); +} + + +static size_t getFileSize(const char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); + if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ +#else + struct stat statbuf; + r = stat(infilename, &statbuf); + if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ +#endif + return (size_t)statbuf.st_size; +} + + +static int isDirectory(const char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); + if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; +#else + struct stat statbuf; + r = stat(infilename, &statbuf); + if (!r && S_ISDIR(statbuf.st_mode)) return 1; +#endif + return 0; +} + + +/** loadFile() : + * requirement : `buffer` size >= `fileSize` */ +static void loadFile(void* buffer, const char* fileName, size_t fileSize) +{ + FILE* const f = fopen(fileName, "rb"); + if (isDirectory(fileName)) { + MSG("Ignoring %s directory \n", fileName); + exit(2); + } + if (f==NULL) { + MSG("Impossible to open %s \n", fileName); + exit(3); + } + { size_t const readSize = fread(buffer, 1, fileSize, f); + if (readSize != fileSize) { + MSG("Error reading %s \n", fileName); + exit(5); + } } + fclose(f); +} + + +static void fileCheck(const char* fileName, int clevel) +{ + size_t const fileSize = getFileSize(fileName); + void* const buffer = malloc(fileSize + !fileSize /* avoid 0 */); + if (!buffer) { + MSG("not enough memory \n"); + exit(4); + } + loadFile(buffer, fileName, fileSize); + roundTripCheck(buffer, fileSize, clevel); + free (buffer); +} + + +int bad_usage(const char* exeName) +{ + MSG(" \n"); + MSG("bad usage: \n"); + MSG(" \n"); + MSG("%s [Options] fileName \n", exeName); + MSG(" \n"); + MSG("Options: \n"); + MSG("-# : use #=[0-9] compression level (default:0 == random) \n"); + return 1; +} + + +int main(int argCount, const char** argv) +{ + const char* const exeName = argv[0]; + int argNb = 1; + int clevel = 0; + + assert(argCount >= 1); + if (argCount < 2) return bad_usage(exeName); + + if (argv[1][0] == '-') { + clevel = argv[1][1] - '0'; + argNb = 2; + } + + if (argNb >= argCount) return bad_usage(exeName); + + fileCheck(argv[argNb], clevel); + MSG("no pb detected \n"); + return 0; +} |