diff options
Diffstat (limited to 'programs')
| -rw-r--r-- | programs/Makefile | 198 | ||||
| -rw-r--r-- | programs/datagencli.c | 190 | ||||
| -rw-r--r-- | programs/frametest.c | 866 | ||||
| -rw-r--r-- | programs/fullbench.c | 813 | ||||
| -rw-r--r-- | programs/fuzzer.c | 1169 |
5 files changed, 2 insertions, 3234 deletions
diff --git a/programs/Makefile b/programs/Makefile index 6214084..8cb416c 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -25,13 +25,6 @@ # lz4 : Command Line Utility, supporting gzip-like arguments # lz4c : CLU, supporting also legacy lz4demo arguments # lz4c32: Same as lz4c, but forced to compile in 32-bits mode -# fuzzer : Test tool, to check lz4 integrity on target platform -# fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode -# frametest : Test tool, to check lz4frame integrity on target platform -# frametest32: Same as frametest, but forced to compile in 32-bits mode -# fullbench : Precisely measure speed for each LZ4 function variant -# fullbench32: Same as fullbench, but forced to compile in 32-bits mode -# datagen : generates synthetic data samples for tests & benchmarks # ########################################################################## DESTDIR ?= @@ -64,9 +57,9 @@ FUZZER_TIME := -T9mn default: lz4 -m32: lz4c32 fullbench32 fuzzer32 frametest32 +m32: lz4c32 -bins: lz4 lz4c fullbench fuzzer frametest datagen +bins: lz4 lz4c all: bins m32 @@ -79,35 +72,10 @@ lz4c : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash. lz4c32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c bench.c lz4io.c lz4cli.c $(CC) -m32 $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT) -fullbench : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o fullbench.c - $(CC) $(FLAGS) $^ -o $@$(EXT) - -fullbench32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c fullbench.c - $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) - -fuzzer : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/xxhash.o fuzzer.c - $(CC) $(FLAGS) $^ -o $@$(EXT) - -fuzzer32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c fuzzer.c - $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) - -frametest: $(LZ4DIR)/lz4frame.o $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/xxhash.o frametest.c - $(CC) $(FLAGS) $^ -o $@$(EXT) - -frametest32: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c frametest.c - $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) - -datagen : datagen.o datagencli.c - $(CC) $(FLAGS) $^ -o $@$(EXT) - clean: @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) @$(RM) core *.o *.test tmp* \ lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) \ - fullbench$(EXT) fullbench32$(EXT) \ - fuzzer$(EXT) fuzzer32$(EXT) \ - frametest$(EXT) frametest32$(EXT) \ - datagen$(EXT) @echo Cleaning completed @@ -141,166 +109,4 @@ uninstall: @$(RM) $(DESTDIR)$(MANDIR)/unlz4.1 @echo lz4 programs successfully uninstalled -test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-mem - -test32: test-lz4c32 test-frametest32 test-fullbench32 test-fuzzer32 test-mem32 - -test-all: test test32 - -test-lz4-sparse: lz4 datagen - @echo "\n ---- test sparse file support ----" - ./datagen -g5M -P100 > tmpSrc - ./lz4 -B4D tmpSrc | ./lz4 -dv --sparse > tmpB4 - diff -s tmpSrc tmpB4 - ./lz4 -B5D tmpSrc | ./lz4 -dv --sparse > tmpB5 - diff -s tmpSrc tmpB5 - ./lz4 -B6D tmpSrc | ./lz4 -dv --sparse > tmpB6 - diff -s tmpSrc tmpB6 - ./lz4 -B7D tmpSrc | ./lz4 -dv --sparse > tmpB7 - diff -s tmpSrc tmpB7 - ./lz4 tmpSrc | ./lz4 -dv --no-sparse > tmpNoSparse - diff -s tmpSrc tmpNoSparse - ls -ls tmp* - ./datagen -s1 -g1200007 -P100 | ./lz4 | ./lz4 -dv --sparse > tmpOdd # Odd size file (to generate non-full last block) - ./datagen -s1 -g1200007 -P100 | diff -s - tmpOdd - ls -ls tmpOdd - @$(RM) tmp* - @echo "\n Compatibility with Console :" - echo "Hello World 1 !" | ./lz4 | ./lz4 -d -c - echo "Hello World 2 !" | ./lz4 | ./lz4 -d | cat - echo "Hello World 3 !" | ./lz4 --no-frame-crc | ./lz4 -d -c - @echo "\n Compatibility with Append :" - ./datagen -P100 -g1M > tmp1M - cat tmp1M tmp1M > tmp2M - ./lz4 -B5 -v tmp1M tmpC - ./lz4 -d -v tmpC tmpR - ./lz4 -d -v tmpC >> tmpR - ls -ls tmp* - diff tmp2M tmpR - @$(RM) tmp* - -test-lz4-contentSize: lz4 datagen - @echo "\n ---- test original size support ----" - ./datagen -g15M > tmp - ./lz4 -v tmp | ./lz4 -t - ./lz4 -v --content-size tmp | ./lz4 -d > tmp2 - diff -s tmp tmp2 - # test large size [2-4] GB - @./datagen -g3G -P100 | ./lz4 | ./lz4 --decompress --force --sparse - tmp - @ls -ls tmp - ./lz4 --quiet --content-size tmp | ./lz4 --verbose --decompress --force --sparse - tmp2 - @ls -ls tmp2 - @$(RM) tmp* - -test-lz4-frame-concatenation: lz4 datagen - @echo "\n ---- test frame concatenation ----" - @echo -n > empty.test - @echo hi > nonempty.test - cat nonempty.test empty.test nonempty.test > orig.test - @./lz4 -zq empty.test > empty.lz4.test - @./lz4 -zq nonempty.test > nonempty.lz4.test - cat nonempty.lz4.test empty.lz4.test nonempty.lz4.test > concat.lz4.test - ./lz4 -d concat.lz4.test > result.test - sdiff orig.test result.test - @$(RM) *.test - @echo frame concatenation test completed - -test-lz4-multiple: lz4 datagen - @echo "\n ---- test multiple files ----" - @./datagen -s1 > tmp1 2> $(VOID) - @./datagen -s2 -g100K > tmp2 2> $(VOID) - @./datagen -s3 -g1M > tmp3 2> $(VOID) - ./lz4 -f -m tmp* - ls -ls tmp* - @$(RM) tmp1 tmp2 tmp3 - ./lz4 -df -m *.lz4 - ls -ls tmp* - ./lz4 -f -m tmp1 notHere tmp2; echo $$? - @$(RM) tmp* - -test-lz4-basic: lz4 datagen - @echo "\n ---- test lz4 basic compression/decompression ----" - ./datagen -g0 | ./lz4 -v | ./lz4 -t - ./datagen -g16KB | ./lz4 -9 | ./lz4 -t - ./datagen -g20KB > tmpSrc - ./lz4 < tmpSrc | ./lz4 -d > tmpRes - diff -q tmpSrc tmpRes - ./lz4 --no-frame-crc < tmpSrc | ./lz4 -d > tmpRes - diff -q tmpSrc tmpRes - ./datagen | ./lz4 | ./lz4 -t - ./datagen -g6M -P99 | ./lz4 -9BD | ./lz4 -t - ./datagen -g17M | ./lz4 -9v | ./lz4 -qt - ./datagen -g33M | ./lz4 --no-frame-crc | ./lz4 -t - ./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -t - ./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -qt - ./datagen -g6GB | ./lz4 -vq9BD | ./lz4 -qt - @$(RM) tmp* - -test-lz4: lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-sparse test-lz4-contentSize test-lz4-frame-concatenation - @echo "\n ---- test pass-through ----" - ./datagen | ./lz4 -tf - -test-lz4c: lz4c datagen - @echo "\n ---- test lz4c version ----" - ./datagen -g256MB | ./lz4c -l -v | ./lz4c -t - -test-interop-32-64: lz4 lz4c32 datagen - @echo "\n ---- test interoperability 32-bits -vs- 64 bits ----" - ./datagen -g16KB | ./lz4c32 -9 | ./lz4 -t - ./datagen -P10 | ./lz4 -9B4 | ./lz4c32 -t - ./datagen | ./lz4c32 | ./lz4 -t - ./datagen -g1M | ./lz4 -3B5 | ./lz4c32 -t - ./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4 -qt - ./datagen -g1G -P90 | ./lz4 | ./lz4c32 -t - ./datagen -g6GB | ./lz4c32 -vq9BD | ./lz4 -qt - -test-lz4c32-basic: lz4c32 datagen - @echo "\n ---- test lz4c32 32-bits version ----" - ./datagen -g16KB | ./lz4c32 -9 | ./lz4c32 -t - ./datagen | ./lz4c32 | ./lz4c32 -t - ./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4c32 -qt - ./datagen -g6GB | ./lz4c32 -vqB5D | ./lz4c32 -qt - -test-lz4c32: test-lz4c32-basic test-interop-32-64 - -test-fullbench: fullbench - ./fullbench --no-prompt $(NB_LOOPS) $(TEST_FILES) - -test-fullbench32: fullbench32 - ./fullbench32 --no-prompt $(NB_LOOPS) $(TEST_FILES) - -test-fuzzer: fuzzer - ./fuzzer $(FUZZER_TIME) - -test-fuzzer32: fuzzer32 - ./fuzzer32 $(FUZZER_TIME) - -test-frametest: frametest - ./frametest $(FUZZER_TIME) - -test-frametest32: frametest32 - ./frametest32 $(FUZZER_TIME) - -test-mem: lz4 datagen fuzzer frametest fullbench - @echo "\n ---- valgrind tests : memory analyzer ----" - valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID) - ./datagen -g16KB > tmp - valgrind --leak-check=yes --error-exitcode=1 ./lz4 -9 -BD -f tmp $(VOID) - ./datagen -g16KB -s2 > tmp2 - ./datagen -g16KB -s3 > tmp3 - valgrind --leak-check=yes --error-exitcode=1 ./lz4 --force --multiple tmp tmp2 tmp3 - ./datagen -g16MB > tmp - valgrind --leak-check=yes --error-exitcode=1 ./lz4 -9 -B5D -f tmp tmp2 - valgrind --leak-check=yes --error-exitcode=1 ./lz4 -t tmp2 - valgrind --leak-check=yes --error-exitcode=1 ./lz4 -bi1 tmp - valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 tmp tmp2 - ./datagen -g256MB > tmp - valgrind --leak-check=yes --error-exitcode=1 ./lz4 -B4D -f -vq tmp $(VOID) - $(RM) tmp* - valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i64 -t1 - valgrind --leak-check=yes --error-exitcode=1 ./frametest -i256 - -test-mem32: lz4c32 datagen -# unfortunately, valgrind doesn't seem to work with non-native binary... - endif diff --git a/programs/datagencli.c b/programs/datagencli.c deleted file mode 100644 index 2fbcc50..0000000 --- a/programs/datagencli.c +++ /dev/null @@ -1,190 +0,0 @@ -/* - datagencli.c - compressible data command line generator - Copyright (C) Yann Collet 2012-2015 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 source repository : https://github.com/Cyan4973/lz4 - - Public forum : https://groups.google.com/forum/#!forum/lz4c -*/ - -/************************************** -* Includes -**************************************/ -#include <stdio.h> /* fprintf, stderr */ -#include "datagen.h" /* RDG_generate */ -#include "lz4.h" /* LZ4_VERSION_STRING */ - - -/************************************** -* Basic Types -**************************************/ -#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# include <stdint.h> - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - - -/************************************** -* Constants -**************************************/ -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -#define SIZE_DEFAULT (64 KB) -#define SEED_DEFAULT 0 -#define COMPRESSIBILITY_DEFAULT 50 - - -/************************************** -* Macros -**************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static unsigned displayLevel = 2; - - -/********************************************************* -* Command line -*********************************************************/ -static int usage(char* programName) -{ - DISPLAY( "Compressible data generator\n"); - DISPLAY( "Usage :\n"); - DISPLAY( " %s [size] [args]\n", programName); - DISPLAY( "\n"); - DISPLAY( "Arguments :\n"); - DISPLAY( " -g# : generate # data (default:%i)\n", SIZE_DEFAULT); - DISPLAY( " -s# : Select seed (default:%i)\n", SEED_DEFAULT); - DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", COMPRESSIBILITY_DEFAULT); - DISPLAY( " -h : display help and exit\n"); - DISPLAY( "Special values :\n"); - DISPLAY( " -P0 : generate incompressible noise\n"); - DISPLAY( " -P100 : generate sparse files\n"); - return 0; -} - - -int main(int argc, char** argv) -{ - int argNb; - double proba = (double)COMPRESSIBILITY_DEFAULT / 100; - double litProba = 0.0; - U64 size = SIZE_DEFAULT; - U32 seed = SEED_DEFAULT; - char* programName; - - /* Check command line */ - programName = argv[0]; - for(argNb=1; argNb<argc; argNb++) - { - char* argument = argv[argNb]; - - if(!argument) continue; /* Protection if argument empty */ - - /* Handle commands. Aggregated commands are allowed */ - if (*argument=='-') - { - argument++; - while (*argument!=0) - { - switch(*argument) - { - case 'h': - return usage(programName); - case 'g': - argument++; - size=0; - while ((*argument>='0') && (*argument<='9')) - { - size *= 10; - size += *argument - '0'; - argument++; - } - if (*argument=='K') { size <<= 10; argument++; } - if (*argument=='M') { size <<= 20; argument++; } - if (*argument=='G') { size <<= 30; argument++; } - if (*argument=='B') { argument++; } - break; - case 's': - argument++; - seed=0; - while ((*argument>='0') && (*argument<='9')) - { - seed *= 10; - seed += *argument - '0'; - argument++; - } - break; - case 'P': - argument++; - proba=0.0; - while ((*argument>='0') && (*argument<='9')) - { - proba *= 10; - proba += *argument - '0'; - argument++; - } - if (proba>100.) proba=100.; - proba /= 100.; - break; - case 'L': /* hidden argument : Literal distribution probability */ - argument++; - litProba=0.; - while ((*argument>='0') && (*argument<='9')) - { - litProba *= 10; - litProba += *argument - '0'; - argument++; - } - if (litProba>100.) litProba=100.; - litProba /= 100.; - break; - case 'v': - displayLevel = 4; - argument++; - break; - default: - return usage(programName); - } - } - - } - } - - DISPLAYLEVEL(4, "Data Generator %s \n", LZ4_VERSION_STRING); - DISPLAYLEVEL(3, "Seed = %u \n", seed); - if (proba!=COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", (U32)(proba*100)); - - RDG_genOut(size, proba, litProba, seed); - DISPLAYLEVEL(1, "\n"); - - return 0; -} diff --git a/programs/frametest.c b/programs/frametest.c deleted file mode 100644 index 4322ab8..0000000 --- a/programs/frametest.c +++ /dev/null @@ -1,866 +0,0 @@ -/* - frameTest - test tool for lz4frame - Copyright (C) Yann Collet 2014-2016 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 homepage : http://www.lz4.org - - LZ4 source repository : https://github.com/Cyan4973/lz4 -*/ - -/*-************************************ -* Compiler specific -**************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ -#endif - - -/*-************************************ -* Includes -**************************************/ -#include <stdlib.h> /* malloc, free */ -#include <stdio.h> /* fprintf */ -#include <string.h> /* strcmp */ -#include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */ -#include "lz4frame_static.h" -#include "lz4.h" /* LZ4_VERSION_STRING */ -#define XXH_STATIC_LINKING_ONLY -#include "xxhash.h" /* XXH64 */ - - -/*-************************************ -* Basic Types -**************************************/ -#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# include <stdint.h> -typedef uint8_t BYTE; -typedef uint16_t U16; -typedef uint32_t U32; -typedef int32_t S32; -typedef uint64_t U64; -#else -typedef unsigned char BYTE; -typedef unsigned short U16; -typedef unsigned int U32; -typedef signed int S32; -typedef unsigned long long U64; -#endif - - -/* unoptimized version; solves endianess & alignment issues */ -static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32) -{ - BYTE* dstPtr = (BYTE*)dstVoidPtr; - dstPtr[0] = (BYTE)value32; - dstPtr[1] = (BYTE)(value32 >> 8); - dstPtr[2] = (BYTE)(value32 >> 16); - dstPtr[3] = (BYTE)(value32 >> 24); -} - - -/*-************************************ -* Constants -**************************************/ -#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U - -#define KB *(1U<<10) -#define MB *(1U<<20) -#define GB *(1U<<30) - -static const U32 nbTestsDefault = 256 KB; -#define COMPRESSIBLE_NOISE_LENGTH (2 MB) -#define FUZ_COMPRESSIBILITY_DEFAULT 50 -static const U32 prime1 = 2654435761U; -static const U32 prime2 = 2246822519U; - - - -/*-************************************ -* Macros -**************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } -#define DISPLAYUPDATE(l, ...) if (displayLevel>=l) { \ - if ((FUZ_GetClockSpan(g_clockTime) > refreshRate) || (displayLevel>=4)) \ - { g_clockTime = clock(); DISPLAY(__VA_ARGS__); \ - if (displayLevel>=4) fflush(stdout); } } -static const clock_t refreshRate = CLOCKS_PER_SEC / 6; -static clock_t g_clockTime = 0; - - -/*-*************************************** -* Local Parameters -*****************************************/ -static U32 no_prompt = 0; -static char* programName; -static U32 displayLevel = 2; -static U32 pause = 0; - - -/*-******************************************************* -* Fuzzer functions -*********************************************************/ -static clock_t FUZ_GetClockSpan(clock_t clockStart) -{ - return clock() - clockStart; /* works even if overflow; max span ~ 30 mn */ -} - - -#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) -unsigned int FUZ_rand(unsigned int* src) -{ - U32 rand32 = *src; - rand32 *= prime1; - rand32 += prime2; - rand32 = FUZ_rotl32(rand32, 13); - *src = rand32; - return rand32 >> 5; -} - - -#define FUZ_RAND15BITS (FUZ_rand(seed) & 0x7FFF) -#define FUZ_RANDLENGTH ( (FUZ_rand(seed) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15) -static void FUZ_fillCompressibleNoiseBuffer(void* buffer, unsigned bufferSize, double proba, U32* seed) -{ - BYTE* BBuffer = (BYTE*)buffer; - unsigned pos = 0; - U32 P32 = (U32)(32768 * proba); - - /* First Byte */ - BBuffer[pos++] = (BYTE)(FUZ_rand(seed)); - - while (pos < bufferSize) { - /* Select : Literal (noise) or copy (within 64K) */ - if (FUZ_RAND15BITS < P32) { - /* Copy (within 64K) */ - unsigned match, end; - unsigned length = FUZ_RANDLENGTH + 4; - unsigned offset = FUZ_RAND15BITS + 1; - if (offset > pos) offset = pos; - if (pos + length > bufferSize) length = bufferSize - pos; - match = pos - offset; - end = pos + length; - while (pos < end) BBuffer[pos++] = BBuffer[match++]; - } else { - /* Literal (noise) */ - unsigned end; - unsigned length = FUZ_RANDLENGTH; - if (pos + length > bufferSize) length = bufferSize - pos; - end = pos + length; - while (pos < end) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5); - } - } -} - - -static unsigned FUZ_highbit(U32 v32) -{ - unsigned nbBits = 0; - if (v32==0) return 0; - while (v32) v32 >>= 1, nbBits ++; - return nbBits; -} - - -int basicTests(U32 seed, double compressibility) -{ - int testResult = 0; - void* CNBuffer; - void* compressedBuffer; - void* decodedBuffer; - U32 randState = seed; - size_t cSize, testSize; - LZ4F_preferences_t prefs; - LZ4F_decompressionContext_t dCtx = NULL; - LZ4F_compressionContext_t cctx = NULL; - U64 crcOrig; - - /* Create compressible test buffer */ - memset(&prefs, 0, sizeof(prefs)); - CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); - compressedBuffer = malloc(LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL)); - decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); - FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); - crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); - - /* Special case : null-content frame */ - testSize = 0; - DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : \n"); - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL); - if (LZ4F_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "Compressed null content into a %i bytes frame \n", (int)cSize); - - DISPLAYLEVEL(3, "LZ4F_createDecompressionContext \n"); - { LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) goto _output_error; } - - DISPLAYLEVEL(3, "LZ4F_getFrameInfo on null-content frame (#157) \n"); - { size_t avail_in = cSize; - LZ4F_frameInfo_t frame_info; - LZ4F_errorCode_t const errorCode = LZ4F_getFrameInfo(dCtx, &frame_info, compressedBuffer, &avail_in); - if (LZ4F_isError(errorCode)) goto _output_error; - } - - DISPLAYLEVEL(3, "LZ4F_freeDecompressionContext \n"); - { LZ4F_errorCode_t const errorCode = LZ4F_freeDecompressionContext(dCtx); - if (LZ4F_isError(errorCode)) goto _output_error; } - - /* Trivial tests : one-step frame */ - testSize = COMPRESSIBLE_NOISE_LENGTH; - DISPLAYLEVEL(3, "LZ4F_compressFrame, using default preferences : \n"); - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL); - if (LZ4F_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - - DISPLAYLEVEL(3, "Decompression test : \n"); - { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; - size_t compressedBufferSize = cSize; - BYTE* op = (BYTE*)decodedBuffer; - BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; - BYTE* ip = (BYTE*)compressedBuffer; - BYTE* const iend = (BYTE*)compressedBuffer + cSize; - U64 crcDest; - - LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) goto _output_error; - - DISPLAYLEVEL(3, "Single Block : \n"); - errorCode = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; - crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); - if (crcDest != crcOrig) goto _output_error; - DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize); - - DISPLAYLEVEL(3, "Reusing decompression context \n"); - { size_t iSize = compressedBufferSize - 4; - const BYTE* cBuff = (const BYTE*) compressedBuffer; - size_t decResult; - DISPLAYLEVEL(3, "Missing last 4 bytes : "); - decResult = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, cBuff, &iSize, NULL); - if (LZ4F_isError(decResult)) goto _output_error; - if (!decResult) goto _output_error; /* not finished */ - DISPLAYLEVEL(3, "indeed, request %u bytes \n", (unsigned)decResult); - cBuff += iSize; - iSize = decResult; - decResult = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, cBuff, &iSize, NULL); - if (decResult != 0) goto _output_error; /* should finish now */ - crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); - if (crcDest != crcOrig) goto _output_error; - } - - { size_t oSize = 0; - size_t iSize = 0; - LZ4F_frameInfo_t fi; - - DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : "); - errorCode = LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; - DISPLAYLEVEL(3, " %u \n", (unsigned)errorCode); - - DISPLAYLEVEL(3, "get FrameInfo on null input : "); - errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize); - if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error; - DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode)); - - DISPLAYLEVEL(3, "get FrameInfo on not enough input : "); - iSize = 6; - errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize); - if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error; - DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode)); - ip += iSize; - - DISPLAYLEVEL(3, "get FrameInfo on enough input : "); - iSize = 15 - iSize; - errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize); - if (LZ4F_isError(errorCode)) goto _output_error; - DISPLAYLEVEL(3, " correctly decoded \n"); - ip += iSize; - } - - DISPLAYLEVEL(3, "Byte after byte : \n"); - while (ip < iend) { - size_t oSize = oend-op; - size_t iSize = 1; - errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; - op += oSize; - ip += iSize; - } - crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); - if (crcDest != crcOrig) goto _output_error; - DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-(BYTE*)decodedBuffer), COMPRESSIBLE_NOISE_LENGTH); - - errorCode = LZ4F_freeDecompressionContext(dCtx); - if (LZ4F_isError(errorCode)) goto _output_error; - } - - DISPLAYLEVEL(3, "Using 64 KB block : \n"); - prefs.frameInfo.blockSizeID = LZ4F_max64KB; - prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - - DISPLAYLEVEL(3, "without checksum : \n"); - prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - - DISPLAYLEVEL(3, "Using 256 KB block : \n"); - prefs.frameInfo.blockSizeID = LZ4F_max256KB; - prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - - DISPLAYLEVEL(3, "Decompression test : \n"); - { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; - unsigned maxBits = FUZ_highbit((U32)decodedBufferSize); - BYTE* op = (BYTE*)decodedBuffer; - BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; - BYTE* ip = (BYTE*)compressedBuffer; - BYTE* const iend = (BYTE*)compressedBuffer + cSize; - U64 crcDest; - - LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) goto _output_error; - - DISPLAYLEVEL(3, "random segment sizes : \n"); - while (ip < iend) { - unsigned nbBits = FUZ_rand(&randState) % maxBits; - size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; - size_t oSize = oend-op; - if (iSize > (size_t)(iend-ip)) iSize = iend-ip; - //DISPLAY("%7i : + %6i\n", (int)(ip-(BYTE*)compressedBuffer), (int)iSize); - errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; - op += oSize; - ip += iSize; - } - crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); - if (crcDest != crcOrig) goto _output_error; - DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize); - - errorCode = LZ4F_freeDecompressionContext(dCtx); - if (LZ4F_isError(errorCode)) goto _output_error; - } - - DISPLAYLEVEL(3, "without checksum : \n"); - prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - - DISPLAYLEVEL(3, "Using 1 MB block : \n"); - prefs.frameInfo.blockSizeID = LZ4F_max1MB; - prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - - DISPLAYLEVEL(3, "without checksum : \n"); - prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - - DISPLAYLEVEL(3, "Using 4 MB block : \n"); - prefs.frameInfo.blockSizeID = LZ4F_max4MB; - prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - - DISPLAYLEVEL(3, "without checksum : \n"); - prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LZ4F_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - - { size_t errorCode; - BYTE* const ostart = (BYTE*)compressedBuffer; - BYTE* op = ostart; - errorCode = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) goto _output_error; - - DISPLAYLEVEL(3, "compress without frameSize : \n"); - memset(&(prefs.frameInfo), 0, sizeof(prefs.frameInfo)); - errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs); - if (LZ4F_isError(errorCode)) goto _output_error; - op += errorCode; - errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; - op += errorCode; - errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart)); - - DISPLAYLEVEL(3, "compress with frameSize : \n"); - prefs.frameInfo.contentSize = testSize; - op = ostart; - errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs); - if (LZ4F_isError(errorCode)) goto _output_error; - op += errorCode; - errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; - op += errorCode; - errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart)); - - DISPLAYLEVEL(3, "compress with wrong frameSize : \n"); - prefs.frameInfo.contentSize = testSize+1; - op = ostart; - errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs); - if (LZ4F_isError(errorCode)) goto _output_error; - op += errorCode; - errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; - op += errorCode; - errorCode = LZ4F_compressEnd(cctx, op, testSize, NULL); - if (LZ4F_isError(errorCode)) { DISPLAYLEVEL(3, "Error correctly detected : %s \n", LZ4F_getErrorName(errorCode)); } - else - goto _output_error; - - errorCode = LZ4F_freeCompressionContext(cctx); - if (LZ4F_isError(errorCode)) goto _output_error; - cctx = NULL; - } - - DISPLAYLEVEL(3, "Skippable frame test : \n"); - { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; - unsigned maxBits = FUZ_highbit((U32)decodedBufferSize); - BYTE* op = (BYTE*)decodedBuffer; - BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; - BYTE* ip = (BYTE*)compressedBuffer; - BYTE* iend = (BYTE*)compressedBuffer + cSize + 8; - - LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) goto _output_error; - - /* generate skippable frame */ - FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START); - FUZ_writeLE32(ip+4, (U32)cSize); - - DISPLAYLEVEL(3, "random segment sizes : \n"); - while (ip < iend) { - unsigned nbBits = FUZ_rand(&randState) % maxBits; - size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; - size_t oSize = oend-op; - if (iSize > (size_t)(iend-ip)) iSize = iend-ip; - errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; - op += oSize; - ip += iSize; - } - DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)decodedBufferSize); - - /* generate zero-size skippable frame */ - DISPLAYLEVEL(3, "zero-size skippable frame\n"); - ip = (BYTE*)compressedBuffer; - op = (BYTE*)decodedBuffer; - FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+1); - FUZ_writeLE32(ip+4, 0); - iend = ip+8; - - while (ip < iend) { - unsigned nbBits = FUZ_rand(&randState) % maxBits; - size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; - size_t oSize = oend-op; - if (iSize > (size_t)(iend-ip)) iSize = iend-ip; - errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; - op += oSize; - ip += iSize; - } - DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8)); - - DISPLAYLEVEL(3, "Skippable frame header complete in first call \n"); - ip = (BYTE*)compressedBuffer; - op = (BYTE*)decodedBuffer; - FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+2); - FUZ_writeLE32(ip+4, 10); - iend = ip+18; - while (ip < iend) { - size_t iSize = 10; - size_t oSize = 10; - if (iSize > (size_t)(iend-ip)) iSize = iend-ip; - errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); - if (LZ4F_isError(errorCode)) goto _output_error; - op += oSize; - ip += iSize; - } - DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8)); - } - - DISPLAY("Basic tests completed \n"); -_end: - free(CNBuffer); - free(compressedBuffer); - free(decodedBuffer); - LZ4F_freeDecompressionContext(dCtx); dCtx = NULL; - LZ4F_freeCompressionContext(cctx); cctx = NULL; - return testResult; - -_output_error: - testResult = 1; - DISPLAY("Error detected ! \n"); - goto _end; -} - - -static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, unsigned nonContiguous) -{ - int p=0; - const BYTE* b1=(const BYTE*)buff1; - const BYTE* b2=(const BYTE*)buff2; - if (nonContiguous) { - DISPLAY("Non-contiguous output test (%i bytes)\n", (int)size); - return; - } - while (b1[p]==b2[p]) p++; - DISPLAY("Error at pos %i/%i : %02X != %02X \n", p, (int)size, b1[p], b2[p]); -} - - -static const U32 srcDataLength = 9 MB; /* needs to be > 2x4MB to test large blocks */ - -int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration_s) -{ - unsigned testResult = 0; - unsigned testNb = 0; - void* srcBuffer = NULL; - void* compressedBuffer = NULL; - void* decodedBuffer = NULL; - U32 coreRand = seed; - LZ4F_decompressionContext_t dCtx = NULL; - LZ4F_compressionContext_t cCtx = NULL; - size_t result; - clock_t const startClock = clock(); - clock_t const clockDuration = duration_s * CLOCKS_PER_SEC; - XXH64_state_t xxh64; -# define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ - DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } - - /* Create buffers */ - result = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); - CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result); - result = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION); - CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result); - srcBuffer = malloc(srcDataLength); - CHECK(srcBuffer==NULL, "srcBuffer Allocation failed"); - compressedBuffer = malloc(LZ4F_compressFrameBound(srcDataLength, NULL)); - CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed"); - decodedBuffer = calloc(1, srcDataLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */ - CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed"); - FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand); - - /* jump to requested testNb */ - for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); // sync randomizer - - /* main fuzzer test loop */ - for ( ; (testNb < nbTests) || (clockDuration > FUZ_GetClockSpan(startClock)) ; testNb++) { - U32 randState = coreRand ^ prime1; - unsigned BSId = 4 + (FUZ_rand(&randState) & 3); - unsigned BMId = FUZ_rand(&randState) & 1; - unsigned CCflag = FUZ_rand(&randState) & 1; - unsigned autoflush = (FUZ_rand(&randState) & 7) == 2; - LZ4F_preferences_t prefs; - LZ4F_compressOptions_t cOptions; - LZ4F_decompressOptions_t dOptions; - unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(srcDataLength-1) - 1)) + 1; - size_t srcSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; - size_t srcStart = FUZ_rand(&randState) % (srcDataLength - srcSize); - U64 frameContentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0; - size_t cSize; - U64 crcOrig; - LZ4F_preferences_t* prefsPtr = &prefs; - - (void)FUZ_rand(&coreRand); /* update seed */ - memset(&prefs, 0, sizeof(prefs)); - memset(&cOptions, 0, sizeof(cOptions)); - memset(&dOptions, 0, sizeof(dOptions)); - prefs.frameInfo.blockMode = (LZ4F_blockMode_t)BMId; - prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)BSId; - prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)CCflag; - prefs.frameInfo.contentSize = frameContentSize; - prefs.autoFlush = autoflush; - prefs.compressionLevel = FUZ_rand(&randState) % 5; - if ((FUZ_rand(&randState) & 0xF) == 1) prefsPtr = NULL; - - DISPLAYUPDATE(2, "\r%5u ", testNb); - crcOrig = XXH64((BYTE*)srcBuffer+srcStart, srcSize, 1); - - if ((FUZ_rand(&randState) & 0xFFF) == 0) { - /* create a skippable frame (rare case) */ - BYTE* op = (BYTE*)compressedBuffer; - FUZ_writeLE32(op, LZ4F_MAGIC_SKIPPABLE_START + (FUZ_rand(&randState) & 15)); - FUZ_writeLE32(op+4, (U32)srcSize); - cSize = srcSize+8; - } else if ((FUZ_rand(&randState) & 0xF) == 2) { - cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(srcSize, prefsPtr), (char*)srcBuffer + srcStart, srcSize, prefsPtr); - CHECK(LZ4F_isError(cSize), "LZ4F_compressFrame failed : error %i (%s)", (int)cSize, LZ4F_getErrorName(cSize)); - } else { - const BYTE* ip = (const BYTE*)srcBuffer + srcStart; - const BYTE* const iend = ip + srcSize; - BYTE* op = (BYTE*)compressedBuffer; - BYTE* const oend = op + LZ4F_compressFrameBound(srcDataLength, NULL); - unsigned maxBits = FUZ_highbit((U32)srcSize); - result = LZ4F_compressBegin(cCtx, op, oend-op, prefsPtr); - CHECK(LZ4F_isError(result), "Compression header failed (error %i)", (int)result); - op += result; - while (ip < iend) { - unsigned nbBitsSeg = FUZ_rand(&randState) % maxBits; - size_t iSize = (FUZ_rand(&randState) & ((1<<nbBitsSeg)-1)) + 1; - size_t oSize = LZ4F_compressBound(iSize, prefsPtr); - unsigned forceFlush = ((FUZ_rand(&randState) & 3) == 1); - if (iSize > (size_t)(iend-ip)) iSize = iend-ip; - cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1); - - result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions); - CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result); - op += result; - ip += iSize; - - if (forceFlush) { - result = LZ4F_flush(cCtx, op, oend-op, &cOptions); - CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result); - op += result; - } - } - result = LZ4F_compressEnd(cCtx, op, oend-op, &cOptions); - CHECK(LZ4F_isError(result), "Compression completion failed (error %i)", (int)result); - op += result; - cSize = op-(BYTE*)compressedBuffer; - } - - { const BYTE* ip = (const BYTE*)compressedBuffer; - const BYTE* const iend = ip + cSize; - BYTE* op = (BYTE*)decodedBuffer; - BYTE* const oend = op + srcDataLength; - size_t totalOut = 0; - unsigned maxBits = FUZ_highbit((U32)cSize); - unsigned nonContiguousDst = (FUZ_rand(&randState) & 3) == 1; /* 0 : contiguous; 1 : non-contiguous; 2 : dst overwritten */ - nonContiguousDst += FUZ_rand(&randState) & nonContiguousDst; /* 0=>0; 1=>1,2 */ - XXH64_reset(&xxh64, 1); - if (maxBits < 3) maxBits = 3; - while (ip < iend) { - unsigned const nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1; - unsigned const nbBitsO = (FUZ_rand(&randState) % (maxBits)) + 1; - size_t iSize = (FUZ_rand(&randState) & ((1<<nbBitsI)-1)) + 1; - size_t oSize = (FUZ_rand(&randState) & ((1<<nbBitsO)-1)) + 2; - if (iSize > (size_t)(iend-ip)) iSize = iend-ip; - if (oSize > (size_t)(oend-op)) oSize = oend-op; - dOptions.stableDst = FUZ_rand(&randState) & 1; - if (nonContiguousDst==2) dOptions.stableDst = 0; - result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions); - if (result == (size_t)-LZ4F_ERROR_contentChecksum_invalid) - locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst); - CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s)", (int)result, LZ4F_getErrorName((LZ4F_errorCode_t)result)); - XXH64_update(&xxh64, op, (U32)oSize); - totalOut += oSize; - op += oSize; - ip += iSize; - op += nonContiguousDst; - if (nonContiguousDst==2) op = (BYTE*)decodedBuffer; /* overwritten destination */ - } - CHECK(result != 0, "Frame decompression failed (error %i)", (int)result); - if (totalOut) { /* otherwise, it's a skippable frame */ - U64 const crcDecoded = XXH64_digest(&xxh64); - if (crcDecoded != crcOrig) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst); - CHECK(crcDecoded != crcOrig, "Decompression corruption"); - } - } - } - - DISPLAYLEVEL(2, "\rAll tests completed \n"); - -_end: - LZ4F_freeDecompressionContext(dCtx); - LZ4F_freeCompressionContext(cCtx); - free(srcBuffer); - free(compressedBuffer); - free(decodedBuffer); - - if (pause) { - DISPLAY("press enter to finish \n"); - (void)getchar(); - } - return testResult; - -_output_error: - testResult = 1; - goto _end; -} - - -int FUZ_usage(void) -{ - DISPLAY( "Usage :\n"); - DISPLAY( " %s [args]\n", programName); - DISPLAY( "\n"); - DISPLAY( "Arguments :\n"); - DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault); - DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n"); - DISPLAY( " -s# : Select seed (default:prompt user)\n"); - DISPLAY( " -t# : Select starting test number (default:0)\n"); - DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); - DISPLAY( " -v : verbose\n"); - DISPLAY( " -h : display help and exit\n"); - return 0; -} - - -int main(int argc, char** argv) -{ - U32 seed=0; - int seedset=0; - int argNb; - int nbTests = nbTestsDefault; - int testNb = 0; - int proba = FUZ_COMPRESSIBILITY_DEFAULT; - int result=0; - U32 duration=0; - - /* Check command line */ - programName = argv[0]; - for(argNb=1; argNb<argc; argNb++) { - char* argument = argv[argNb]; - - if(!argument) continue; /* Protection if argument empty */ - - /* Decode command (note : aggregated commands are allowed) */ - if (argument[0]=='-') { - if (!strcmp(argument, "--no-prompt")) { - no_prompt=1; - seedset=1; - displayLevel=1; - continue; - } - argument++; - - while (*argument!=0) { - switch(*argument) - { - case 'h': - return FUZ_usage(); - case 'v': - argument++; - displayLevel=4; - break; - case 'q': - argument++; - displayLevel--; - break; - case 'p': /* pause at the end */ - argument++; - pause = 1; - break; - - case 'i': - argument++; - nbTests=0; duration=0; - while ((*argument>='0') && (*argument<='9')) { - nbTests *= 10; - nbTests += *argument - '0'; - argument++; - } - break; - - case 'T': - argument++; - nbTests = 0; duration = 0; - for (;;) { - switch(*argument) - { - case 'm': duration *= 60; argument++; continue; - case 's': - case 'n': argument++; continue; - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': duration *= 10; duration += *argument++ - '0'; continue; - } - break; - } - break; - - case 's': - argument++; - seed=0; - seedset=1; - while ((*argument>='0') && (*argument<='9')) { - seed *= 10; - seed += *argument - '0'; - argument++; - } - break; - case 't': - argument++; - testNb=0; - while ((*argument>='0') && (*argument<='9')) { - testNb *= 10; - testNb += *argument - '0'; - argument++; - } - break; - case 'P': /* compressibility % */ - argument++; - proba=0; - while ((*argument>='0') && (*argument<='9')) { - proba *= 10; - proba += *argument - '0'; - argument++; - } - if (proba<0) proba=0; - if (proba>100) proba=100; - break; - default: - ; - return FUZ_usage(); - } - } - } - } - - /* Get Seed */ - printf("Starting lz4frame tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION_STRING); - - if (!seedset) { - time_t const t = time(NULL); - U32 const h = XXH32(&t, sizeof(t), 1); - seed = h % 10000; - } - printf("Seed = %u\n", seed); - if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba); - - if (nbTests<=0) nbTests=1; - - if (testNb==0) result = basicTests(seed, ((double)proba) / 100); - if (result) return 1; - return fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, duration); -} diff --git a/programs/fullbench.c b/programs/fullbench.c deleted file mode 100644 index ab6bba1..0000000 --- a/programs/fullbench.c +++ /dev/null @@ -1,813 +0,0 @@ -/* - bench.c - Demo program to benchmark open-source compression algorithm - Copyright (C) Yann Collet 2012-2015 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 source repository : https://github.com/Cyan4973/lz4 - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c -*/ - -/************************************** -* Compiler Options -**************************************/ -/* Disable some Visual warning messages */ -#define _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ - -/* Unix Large Files support (>4GB) */ -#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions -# define _LARGEFILE_SOURCE -# define _FILE_OFFSET_BITS 64 -#elif ! defined(__LP64__) // No point defining Large file for 64 bit -# define _LARGEFILE64_SOURCE -#endif - -// S_ISREG & gettimeofday() are not supported by MSVC -#if defined(_MSC_VER) || defined(_WIN32) -# define BMK_LEGACY_TIMER 1 -#endif - - -/************************************** -* Includes -**************************************/ -#include <stdlib.h> /* malloc, free */ -#include <stdio.h> /* fprintf, fopen, ftello64 */ -#include <sys/types.h> /* stat64 */ -#include <sys/stat.h> /* stat64 */ -#include <string.h> /* strcmp */ -#include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */ - -#include "lz4.h" -#include "lz4hc.h" -#include "lz4frame.h" - -#include "xxhash.h" - - -/************************************** -* Compiler Options -**************************************/ -/* S_ISREG & gettimeofday() are not supported by MSVC */ -#if !defined(S_ISREG) -# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) -#endif - - -/************************************** -* Basic Types -**************************************/ -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -# include <stdint.h> - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - - -/************************************** -* Constants -**************************************/ -#define PROGRAM_DESCRIPTION "LZ4 speed analyzer" -#define AUTHOR "Yann Collet" -#define WELCOME_MESSAGE "*** %s v%s %i-bits, by %s ***\n", PROGRAM_DESCRIPTION, LZ4_VERSION_STRING, (int)(sizeof(void*)*8), AUTHOR - -#define NBLOOPS 6 -#define TIMELOOP (CLOCKS_PER_SEC * 25 / 10) - -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -#define KNUTH 2654435761U -#define MAX_MEM (1920 MB) -#define DEFAULT_CHUNKSIZE (4 MB) - -#define ALL_COMPRESSORS 0 -#define ALL_DECOMPRESSORS 0 - - -/************************************** -* Local structures -**************************************/ -struct chunkParameters -{ - U32 id; - char* origBuffer; - char* compressedBuffer; - int origSize; - int compressedSize; -}; - - -/************************************** -* Macros -**************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define PROGRESS(...) g_noPrompt ? 0 : DISPLAY(__VA_ARGS__) - - -/************************************** -* Benchmark Parameters -**************************************/ -static int g_chunkSize = DEFAULT_CHUNKSIZE; -static int g_nbIterations = NBLOOPS; -static int g_pause = 0; -static int g_compressionTest = 1; -static int g_compressionAlgo = ALL_COMPRESSORS; -static int g_decompressionTest = 1; -static int g_decompressionAlgo = ALL_DECOMPRESSORS; -static int g_noPrompt = 0; - -static void BMK_setBlocksize(int bsize) -{ - g_chunkSize = bsize; - DISPLAY("-Using Block Size of %i KB-\n", g_chunkSize>>10); -} - -static void BMK_setNbIterations(int nbLoops) -{ - g_nbIterations = nbLoops; - DISPLAY("- %i iterations -\n", g_nbIterations); -} - -static void BMK_setPause(void) -{ - g_pause = 1; -} - - -/********************************************************* -* Private functions -*********************************************************/ -static clock_t BMK_GetClockSpan( clock_t clockStart ) -{ - return clock() - clockStart; /* works even if overflow; max span ~30 mn */ -} - - -static size_t BMK_findMaxMem(U64 requiredMem) -{ - size_t step = 64 MB; - BYTE* testmem=NULL; - - requiredMem = (((requiredMem >> 26) + 1) << 26); - requiredMem += 2*step; - if (requiredMem > MAX_MEM) requiredMem = MAX_MEM; - - while (!testmem) { - if (requiredMem > step) requiredMem -= step; - else requiredMem >>= 1; - testmem = (BYTE*) malloc ((size_t)requiredMem); - } - free (testmem); - - /* keep some space available */ - if (requiredMem > step) requiredMem -= step; - else requiredMem >>= 1; - - return (size_t)requiredMem; -} - - -static U64 BMK_GetFileSize(const char* infilename) -{ - int r; -#if defined(_MSC_VER) - struct _stat64 statbuf; - r = _stat64(infilename, &statbuf); -#else - struct stat statbuf; - r = stat(infilename, &statbuf); -#endif - if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ - return (U64)statbuf.st_size; -} - - -/********************************************************* -* Benchmark function -*********************************************************/ -static LZ4_stream_t LZ4_stream; -static void local_LZ4_resetDictT(void) -{ - LZ4_resetStream(&LZ4_stream); -} - -static void local_LZ4_createStream(void) -{ - LZ4_resetStream(&LZ4_stream); -} - -static int local_LZ4_saveDict(const char* in, char* out, int inSize) -{ - (void)in; - return LZ4_saveDict(&LZ4_stream, out, inSize); -} - -static int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize) -{ - return LZ4_compress_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)-1); -} - -static int local_LZ4_compress_default_large(const char* in, char* out, int inSize) -{ - return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize)); -} - -static int local_LZ4_compress_default_small(const char* in, char* out, int inSize) -{ - return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize)-1); -} - -static int local_LZ4_compress_fast0(const char* in, char* out, int inSize) -{ - return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 0); -} - -static int local_LZ4_compress_fast1(const char* in, char* out, int inSize) -{ - return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 1); -} - -static int local_LZ4_compress_fast2(const char* in, char* out, int inSize) -{ - return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 2); -} - -static int local_LZ4_compress_fast17(const char* in, char* out, int inSize) -{ - return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 17); -} - -static int local_LZ4_compress_fast_extState0(const char* in, char* out, int inSize) -{ - return LZ4_compress_fast_extState(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize), 0); -} - -static int local_LZ4_compress_fast_continue0(const char* in, char* out, int inSize) -{ - return LZ4_compress_fast_continue(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize), 0); -} - -static int local_LZ4_compress_withState(const char* in, char* out, int inSize) -{ - return LZ4_compress_withState(&LZ4_stream, in, out, inSize); -} - -static int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize) -{ - return LZ4_compress_limitedOutput_withState(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize)-1); -} - -static int local_LZ4_compress_continue(const char* in, char* out, int inSize) -{ - return LZ4_compress_continue(&LZ4_stream, in, out, inSize); -} - -static int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize) -{ - return LZ4_compress_limitedOutput_continue(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize)-1); -} - -/* declare hidden function */ -int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize); - -static int local_LZ4_compress_forceDict(const char* in, char* out, int inSize) -{ - return LZ4_compress_forceExtDict(&LZ4_stream, in, out, inSize); -} - - -/* HC compression functions */ -LZ4_streamHC_t LZ4_streamHC; -static void local_LZ4_resetStreamHC(void) -{ - LZ4_resetStreamHC(&LZ4_streamHC, 0); -} - -static int local_LZ4_saveDictHC(const char* in, char* out, int inSize) -{ - (void)in; - return LZ4_saveDictHC(&LZ4_streamHC, out, inSize); -} - -static int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize) -{ - return LZ4_compressHC_withStateHC(&LZ4_streamHC, in, out, inSize); -} - -static int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize) -{ - return LZ4_compressHC_limitedOutput_withStateHC(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize)-1); -} - -static int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize) -{ - return LZ4_compressHC_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)-1); -} - -static int local_LZ4_compressHC_continue(const char* in, char* out, int inSize) -{ - return LZ4_compressHC_continue(&LZ4_streamHC, in, out, inSize); -} - -static int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize) -{ - return LZ4_compressHC_limitedOutput_continue(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize)-1); -} - - -/* decompression functions */ -static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize) -{ - (void)inSize; - LZ4_decompress_fast(in, out, outSize); - return outSize; -} - -static int local_LZ4_decompress_fast_usingDict(const char* in, char* out, int inSize, int outSize) -{ - (void)inSize; - LZ4_decompress_fast_usingDict(in, out, outSize, out - 65536, 65536); - return outSize; -} - -static int local_LZ4_decompress_safe_usingDict(const char* in, char* out, int inSize, int outSize) -{ - (void)inSize; - LZ4_decompress_safe_usingDict(in, out, inSize, outSize, out - 65536, 65536); - return outSize; -} - -extern int LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize, const char* dict, int dictSize); - -static int local_LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize) -{ - (void)inSize; - LZ4_decompress_safe_forceExtDict(in, out, inSize, outSize, out - 65536, 65536); - return outSize; -} - -static int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSize, int outSize) -{ - return LZ4_decompress_safe_partial(in, out, inSize, outSize - 5, outSize); -} - - -/* frame functions */ -static int local_LZ4F_compressFrame(const char* in, char* out, int inSize) -{ - return (int)LZ4F_compressFrame(out, 2*inSize + 16, in, inSize, NULL); -} - -static LZ4F_decompressionContext_t g_dCtx; - -static int local_LZ4F_decompress(const char* in, char* out, int inSize, int outSize) -{ - size_t srcSize = inSize; - size_t dstSize = outSize; - size_t result; - result = LZ4F_decompress(g_dCtx, out, &dstSize, in, &srcSize, NULL); - if (result!=0) { DISPLAY("Error decompressing frame : unfinished frame\n"); exit(8); } - if (srcSize != (size_t)inSize) { DISPLAY("Error decompressing frame : read size incorrect\n"); exit(9); } - return (int)dstSize; -} - - -#define NB_COMPRESSION_ALGORITHMS 100 -#define NB_DECOMPRESSION_ALGORITHMS 100 -int fullSpeedBench(const char** fileNamesTable, int nbFiles) -{ - int fileIdx=0; - - /* Init */ - { size_t const errorCode = LZ4F_createDecompressionContext(&g_dCtx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) { DISPLAY("dctx allocation issue \n"); return 10; } } - - /* Loop for each fileName */ - while (fileIdx<nbFiles) { - char* orig_buff = NULL; - struct chunkParameters* chunkP = NULL; - char* compressed_buff=NULL; - const char* const inFileName = fileNamesTable[fileIdx++]; - FILE* const inFile = fopen( inFileName, "rb" ); - U64 inFileSize; - size_t benchedSize; - int nbChunks; - int maxCompressedChunkSize; - size_t readSize; - int compressedBuffSize; - U32 crcOriginal; - size_t errorCode; - - /* Check file existence */ - if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } - - /* Memory size adjustments */ - inFileSize = BMK_GetFileSize(inFileName); - if (inFileSize==0) { DISPLAY( "file is empty\n"); fclose(inFile); return 11; } - benchedSize = BMK_findMaxMem(inFileSize*2) / 2; /* because 2 buffers */ - if (benchedSize==0) { DISPLAY( "not enough memory\n"); fclose(inFile); return 11; } - if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; - if (benchedSize < inFileSize) - DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); - - /* Allocation */ - chunkP = (struct chunkParameters*) malloc(((benchedSize / (size_t)g_chunkSize)+1) * sizeof(struct chunkParameters)); - orig_buff = (char*) malloc(benchedSize); - nbChunks = (int) ((benchedSize + (g_chunkSize-1)) / g_chunkSize); - maxCompressedChunkSize = LZ4_compressBound(g_chunkSize); - compressedBuffSize = nbChunks * maxCompressedChunkSize; - compressed_buff = (char*)malloc((size_t)compressedBuffSize); - if(!chunkP || !orig_buff || !compressed_buff) { - DISPLAY("\nError: not enough memory!\n"); - fclose(inFile); - free(orig_buff); - free(compressed_buff); - free(chunkP); - return(12); - } - - /* Fill in src buffer */ - DISPLAY("Loading %s... \r", inFileName); - readSize = fread(orig_buff, 1, benchedSize, inFile); - fclose(inFile); - - if (readSize != benchedSize) { - DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); - free(orig_buff); - free(compressed_buff); - free(chunkP); - return 13; - } - - /* Calculating input Checksum */ - crcOriginal = XXH32(orig_buff, benchedSize,0); - - - /* Bench */ - { int loopNb, nb_loops, chunkNb, cAlgNb, dAlgNb; - size_t cSize=0; - double ratio=0.; - - DISPLAY("\r%79s\r", ""); - DISPLAY(" %s : \n", inFileName); - - /* Bench Compression Algorithms */ - for (cAlgNb=0; (cAlgNb <= NB_COMPRESSION_ALGORITHMS) && (g_compressionTest); cAlgNb++) { - const char* compressorName; - int (*compressionFunction)(const char*, char*, int); - void (*initFunction)(void) = NULL; - double bestTime = 100000000.; - - /* filter compressionAlgo only */ - if ((g_compressionAlgo != ALL_COMPRESSORS) && (g_compressionAlgo != cAlgNb)) continue; - - /* Init data chunks */ - { int i; - size_t remaining = benchedSize; - char* in = orig_buff; - char* out = compressed_buff; - nbChunks = (int) (((int)benchedSize + (g_chunkSize-1))/ g_chunkSize); - for (i=0; i<nbChunks; i++) { - chunkP[i].id = i; - chunkP[i].origBuffer = in; in += g_chunkSize; - if ((int)remaining > g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } - chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; - chunkP[i].compressedSize = 0; - } - } - - switch(cAlgNb) - { - case 0 : DISPLAY("Compression functions : \n"); continue; - case 1 : compressionFunction = local_LZ4_compress_default_large; compressorName = "LZ4_compress_default"; break; - case 2 : compressionFunction = local_LZ4_compress_default_small; compressorName = "LZ4_compress_default(small dst)"; break; - case 3 : compressionFunction = local_LZ4_compress_fast0; compressorName = "LZ4_compress_fast(0)"; break; - case 4 : compressionFunction = local_LZ4_compress_fast1; compressorName = "LZ4_compress_fast(1)"; break; - case 5 : compressionFunction = local_LZ4_compress_fast2; compressorName = "LZ4_compress_fast(2)"; break; - case 6 : compressionFunction = local_LZ4_compress_fast17; compressorName = "LZ4_compress_fast(17)"; break; - case 7 : compressionFunction = local_LZ4_compress_fast_extState0; compressorName = "LZ4_compress_fast_extState(0)"; break; - case 8 : compressionFunction = local_LZ4_compress_fast_continue0; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_fast_continue(0)"; break; - - case 10: compressionFunction = LZ4_compressHC; compressorName = "LZ4_compressHC"; break; - case 11: compressionFunction = local_LZ4_compressHC_limitedOutput; compressorName = "LZ4_compressHC_limitedOutput"; break; - case 12 : compressionFunction = local_LZ4_compressHC_withStateHC; compressorName = "LZ4_compressHC_withStateHC"; break; - case 13: compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; compressorName = "LZ4_compressHC_limitedOutput_withStateHC"; break; - case 14: compressionFunction = local_LZ4_compressHC_continue; initFunction = local_LZ4_resetStreamHC; compressorName = "LZ4_compressHC_continue"; break; - case 15: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = local_LZ4_resetStreamHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break; - case 20: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break; - case 30: compressionFunction = local_LZ4F_compressFrame; compressorName = "LZ4F_compressFrame"; - chunkP[0].origSize = (int)benchedSize; nbChunks=1; - break; - case 40: compressionFunction = local_LZ4_saveDict; compressorName = "LZ4_saveDict"; - LZ4_loadDict(&LZ4_stream, chunkP[0].origBuffer, chunkP[0].origSize); - break; - case 41: compressionFunction = local_LZ4_saveDictHC; compressorName = "LZ4_saveDictHC"; - LZ4_loadDictHC(&LZ4_streamHC, chunkP[0].origBuffer, chunkP[0].origSize); - break; - case 60: DISPLAY("Obsolete compression functions : \n"); continue; - case 61: compressionFunction = LZ4_compress; compressorName = "LZ4_compress"; break; - case 62: compressionFunction = local_LZ4_compress_limitedOutput; compressorName = "LZ4_compress_limitedOutput"; break; - case 63: compressionFunction = local_LZ4_compress_withState; compressorName = "LZ4_compress_withState"; break; - case 64: compressionFunction = local_LZ4_compress_limitedOutput_withState; compressorName = "LZ4_compress_limitedOutput_withState"; break; - case 65: compressionFunction = local_LZ4_compress_continue; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_continue"; break; - case 66: compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_limitedOutput_continue"; break; - default : - continue; /* unknown ID : just skip */ - } - - for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) { - double averageTime; - clock_t clockTime; - - PROGRESS("%1i- %-28.28s :%9i ->\r", loopNb, compressorName, (int)benchedSize); - { size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } /* warming up memory */ - - nb_loops = 0; - clockTime = clock(); - while(clock() == clockTime); - clockTime = clock(); - while(BMK_GetClockSpan(clockTime) < TIMELOOP) { - if (initFunction!=NULL) initFunction(); - for (chunkNb=0; chunkNb<nbChunks; chunkNb++) { - chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize); - if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", compressorName), exit(1); - } - nb_loops++; - } - clockTime = BMK_GetClockSpan(clockTime); - - nb_loops += !nb_loops; /* avoid division by zero */ - averageTime = ((double)clockTime) / nb_loops / CLOCKS_PER_SEC; - if (averageTime < bestTime) bestTime = averageTime; - cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize; - ratio = (double)cSize/(double)benchedSize*100.; - PROGRESS("%1i- %-28.28s :%9i ->%9i (%5.2f%%),%7.1f MB/s\r", loopNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000000); - } - - if (ratio<100.) - DISPLAY("%2i-%-28.28s :%9i ->%9i (%5.2f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000000); - else - DISPLAY("%2i-%-28.28s :%9i ->%9i (%5.1f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 100000); - } - - /* Prepare layout for decompression */ - /* Init data chunks */ - { int i; - size_t remaining = benchedSize; - char* in = orig_buff; - char* out = compressed_buff; - - nbChunks = (int) (((int)benchedSize + (g_chunkSize-1))/ g_chunkSize); - for (i=0; i<nbChunks; i++) { - chunkP[i].id = i; - chunkP[i].origBuffer = in; in += g_chunkSize; - if ((int)remaining > g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } - chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; - chunkP[i].compressedSize = 0; - } - } - for (chunkNb=0; chunkNb<nbChunks; chunkNb++) { - chunkP[chunkNb].compressedSize = LZ4_compress(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize); - if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress"), exit(1); - } - - /* Decompression Algorithms */ - for (dAlgNb=0; (dAlgNb <= NB_DECOMPRESSION_ALGORITHMS) && (g_decompressionTest); dAlgNb++) { - const char* dName; - int (*decompressionFunction)(const char*, char*, int, int); - double bestTime = 100000000.; - - if ((g_decompressionAlgo != ALL_DECOMPRESSORS) && (g_decompressionAlgo != dAlgNb)) continue; - - switch(dAlgNb) - { - case 0: DISPLAY("Decompression functions : \n"); continue; - case 1: decompressionFunction = local_LZ4_decompress_fast; dName = "LZ4_decompress_fast"; break; - case 3: decompressionFunction = local_LZ4_decompress_fast_usingDict; dName = "LZ4_decompress_fast_usingDict"; break; - case 4: decompressionFunction = LZ4_decompress_safe; dName = "LZ4_decompress_safe"; break; - case 6: decompressionFunction = local_LZ4_decompress_safe_usingDict; dName = "LZ4_decompress_safe_usingDict"; break; - case 7: decompressionFunction = local_LZ4_decompress_safe_partial; dName = "LZ4_decompress_safe_partial"; break; - case 8: decompressionFunction = local_LZ4_decompress_safe_forceExtDict; dName = "LZ4_decompress_safe_forceExtDict"; break; - case 9: decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress"; - errorCode = LZ4F_compressFrame(compressed_buff, compressedBuffSize, orig_buff, benchedSize, NULL); - if (LZ4F_isError(errorCode)) { - DISPLAY("Error while preparing compressed frame\n"); - free(orig_buff); - free(compressed_buff); - free(chunkP); - return 1; - } - chunkP[0].origSize = (int)benchedSize; - chunkP[0].compressedSize = (int)errorCode; - nbChunks = 1; - break; - default : - continue; /* skip if unknown ID */ - } - - { size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } /* zeroing source area, for CRC checking */ - - for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) { - double averageTime; - clock_t clockTime; - U32 crcDecoded; - - PROGRESS("%1i- %-29.29s :%10i ->\r", loopNb, dName, (int)benchedSize); - - nb_loops = 0; - clockTime = clock(); - while(clock() == clockTime); - clockTime = clock(); - while(BMK_GetClockSpan(clockTime) < TIMELOOP) { - for (chunkNb=0; chunkNb<nbChunks; chunkNb++) { - int decodedSize = decompressionFunction(chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedSize, chunkP[chunkNb].origSize); - if (chunkP[chunkNb].origSize != decodedSize) DISPLAY("ERROR ! %s() == %i != %i !! \n", dName, decodedSize, chunkP[chunkNb].origSize), exit(1); - } - nb_loops++; - } - clockTime = BMK_GetClockSpan(clockTime); - - nb_loops += !nb_loops; /* Avoid division by zero */ - averageTime = (double)clockTime / nb_loops / CLOCKS_PER_SEC; - if (averageTime < bestTime) bestTime = averageTime; - - PROGRESS("%1i- %-29.29s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000000); - - /* CRC Checking */ - crcDecoded = XXH32(orig_buff, (int)benchedSize, 0); - if (crcOriginal!=crcDecoded) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOriginal, (unsigned)crcDecoded); exit(1); } - } - - DISPLAY("%2i-%-29.29s :%10i -> %7.1f MB/s\n", dAlgNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000000); - } - } - free(orig_buff); - free(compressed_buff); - free(chunkP); - } - - LZ4F_freeDecompressionContext(g_dCtx); - if (g_pause) { printf("press enter...\n"); (void)getchar(); } - - return 0; -} - - -static int usage(const char* exename) -{ - DISPLAY( "Usage :\n"); - DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename); - DISPLAY( "Arguments :\n"); - DISPLAY( " -c : compression tests only\n"); - DISPLAY( " -d : decompression tests only\n"); - DISPLAY( " -H/-h : Help (this text + advanced options)\n"); - return 0; -} - -static int usage_advanced(void) -{ - DISPLAY( "\nAdvanced options :\n"); - DISPLAY( " -c# : test only compression function # [1-%i]\n", NB_COMPRESSION_ALGORITHMS); - DISPLAY( " -d# : test only decompression function # [1-%i]\n", NB_DECOMPRESSION_ALGORITHMS); - DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS); - DISPLAY( " -B# : Block size [4-7](default : 7)\n"); - return 0; -} - -static int badusage(const char* exename) -{ - DISPLAY("Wrong parameters\n"); - usage(exename); - return 0; -} - -int main(int argc, const char** argv) -{ - int i, - filenamesStart=2; - const char* exename = argv[0]; - const char* input_filename=0; - - // Welcome message - DISPLAY(WELCOME_MESSAGE); - - if (argc<2) { badusage(exename); return 1; } - - for(i=1; i<argc; i++) { - const char* argument = argv[i]; - - if(!argument) continue; // Protection if argument empty - if (!strcmp(argument, "--no-prompt")) { - g_noPrompt = 1; - continue; - } - - // Decode command (note : aggregated commands are allowed) - if (argument[0]=='-') { - while (argument[1]!=0) { - argument ++; - - switch(argument[0]) - { - // Select compression algorithm only - case 'c': - g_decompressionTest = 0; - while ((argument[1]>= '0') && (argument[1]<= '9')) { - g_compressionAlgo *= 10; - g_compressionAlgo += argument[1] - '0'; - argument++; - } - break; - - // Select decompression algorithm only - case 'd': - g_compressionTest = 0; - while ((argument[1]>= '0') && (argument[1]<= '9')) { - g_decompressionAlgo *= 10; - g_decompressionAlgo += argument[1] - '0'; - argument++; - } - break; - - // Display help on usage - case 'h' : - case 'H': usage(exename); usage_advanced(); return 0; - - // Modify Block Properties - case 'B': - while (argument[1]!=0) - switch(argument[1]) - { - case '4': - case '5': - case '6': - case '7': - { int B = argument[1] - '0'; - int S = 1 << (8 + 2*B); - BMK_setBlocksize(S); - argument++; - break; - } - case 'D': argument++; break; - default : goto _exit_blockProperties; - } -_exit_blockProperties: - break; - - // Modify Nb Iterations - case 'i': - if ((argument[1] >='0') && (argument[1] <='9')) { - int iters = argument[1] - '0'; - BMK_setNbIterations(iters); - argument++; - } - break; - - // Pause at the end (hidden option) - case 'p': BMK_setPause(); break; - - // Unknown command - default : badusage(exename); return 1; - } - } - continue; - } - - // first provided filename is input - if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } - - } - - // No input filename ==> Error - if(!input_filename) { badusage(exename); return 1; } - - return fullSpeedBench(argv+filenamesStart, argc-filenamesStart); - -} diff --git a/programs/fuzzer.c b/programs/fuzzer.c deleted file mode 100644 index db7ad07..0000000 --- a/programs/fuzzer.c +++ /dev/null @@ -1,1169 +0,0 @@ -/* - fuzzer.c - Fuzzer test tool for LZ4 - Copyright (C) Yann Collet 2012-2016 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 homepage : http://www.lz4.org - - LZ4 source repo : https://github.com/Cyan4973/lz4 -*/ - -/*-************************************ -* Compiler options -**************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS /* fgets */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ -# pragma warning(disable : 4310) /* disable: C4310: constant char value > 127 */ -#endif - -/* S_ISREG & gettimeofday() are not supported by MSVC */ -#if defined(_MSC_VER) || defined(_WIN32) -# define FUZ_LEGACY_TIMER 1 -#endif - - -/*-************************************ -* Includes -**************************************/ -#include <stdlib.h> -#include <stdio.h> /* fgets, sscanf */ -#include <string.h> /* strcmp */ -#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */ -#include "lz4.h" /* LZ4_VERSION_STRING */ -#include "lz4hc.h" -#define XXH_STATIC_LINKING_ONLY -#include "xxhash.h" - - -/*-************************************ -* Basic Types -**************************************/ -#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# include <stdint.h> -typedef uint8_t BYTE; -typedef uint16_t U16; -typedef uint32_t U32; -typedef int32_t S32; -typedef uint64_t U64; -#else -typedef unsigned char BYTE; -typedef unsigned short U16; -typedef unsigned int U32; -typedef signed int S32; -typedef unsigned long long U64; -#endif - - -/*-************************************ -* Constants -**************************************/ -#define NB_ATTEMPTS (1<<16) -#define COMPRESSIBLE_NOISE_LENGTH (1 << 21) -#define FUZ_MAX_BLOCK_SIZE (1 << 17) -#define FUZ_MAX_DICT_SIZE (1 << 15) -#define FUZ_COMPRESSIBILITY_DEFAULT 60 -#define PRIME1 2654435761U -#define PRIME2 2246822519U -#define PRIME3 3266489917U - -#define KB *(1U<<10) -#define MB *(1U<<20) -#define GB *(1U<<30) - - -/*-*************************************** -* Macros -*****************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static int g_displayLevel = 2; -static const clock_t g_refreshRate = CLOCKS_PER_SEC * 25 / 100; -static clock_t g_time = 0; - - -/*-******************************************************* -* Fuzzer functions -*********************************************************/ -static clock_t FUZ_GetClockSpan(clock_t clockStart) -{ - return clock() - clockStart; /* works even if overflow; max span ~ 30mn */ -} - -static U32 FUZ_rotl32(U32 u32, U32 nbBits) -{ - return ((u32 << nbBits) | (u32 >> (32 - nbBits))); -} - -static U32 FUZ_rand(U32* src) -{ - U32 rand32 = *src; - rand32 *= PRIME1; - rand32 ^= PRIME2; - rand32 = FUZ_rotl32(rand32, 13); - *src = rand32; - return rand32; -} - - -#define FUZ_RAND15BITS ((FUZ_rand(seed) >> 3) & 32767) -#define FUZ_RANDLENGTH ( ((FUZ_rand(seed) >> 7) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15) -static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, double proba, U32* seed) -{ - BYTE* BBuffer = (BYTE*)buffer; - size_t pos = 0; - U32 P32 = (U32)(32768 * proba); - - /* First Bytes */ - while (pos < 20) - BBuffer[pos++] = (BYTE)(FUZ_rand(seed)); - - while (pos < bufferSize) - { - /* Select : Literal (noise) or copy (within 64K) */ - if (FUZ_RAND15BITS < P32) - { - /* Copy (within 64K) */ - size_t match, d; - size_t length = FUZ_RANDLENGTH + 4; - size_t offset = FUZ_RAND15BITS + 1; - while (offset > pos) offset >>= 1; - d = pos + length; - while (d > bufferSize) d = bufferSize; - match = pos - offset; - while (pos < d) BBuffer[pos++] = BBuffer[match++]; - } - else - { - /* Literal (noise) */ - size_t d; - size_t length = FUZ_RANDLENGTH; - d = pos + length; - if (d > bufferSize) d = bufferSize; - while (pos < d) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5); - } - } -} - - -#define MAX_NB_BUFF_I134 150 -#define BLOCKSIZE_I134 (32 MB) -/*! FUZ_AddressOverflow() : -* Aggressively pushes memory allocation limits, -* and generates patterns which create address space overflow. -* only possible in 32-bits mode */ -static int FUZ_AddressOverflow(void) -{ - char* buffers[MAX_NB_BUFF_I134+1]; - int i, nbBuff=0; - int highAddress = 0; - - DISPLAY("Overflow tests : "); - - /* Only possible in 32-bits */ - if (sizeof(void*)==8) - { - DISPLAY("64 bits mode : no overflow \n"); - fflush(stdout); - return 0; - } - - buffers[0] = (char*)malloc(BLOCKSIZE_I134); - buffers[1] = (char*)malloc(BLOCKSIZE_I134); - if ((!buffers[0]) || (!buffers[1])) { - DISPLAY("not enough memory for tests \n"); - return 0; - } - - for (nbBuff=2; nbBuff < MAX_NB_BUFF_I134; nbBuff++) { - DISPLAY("%3i \b\b\b\b", nbBuff); - buffers[nbBuff] = (char*)malloc(BLOCKSIZE_I134); - if (buffers[nbBuff]==NULL) goto _endOfTests; - - if (((size_t)buffers[nbBuff] > (size_t)0x80000000) && (!highAddress)) { - DISPLAY("high address detected : "); - fflush(stdout); - highAddress=1; - } - - { size_t const sizeToGenerateOverflow = (size_t)(- ((size_t)buffers[nbBuff-1]) + 512); - int const nbOf255 = (int)((sizeToGenerateOverflow / 255) + 1); - char* const input = buffers[nbBuff-1]; - char* output = buffers[nbBuff]; - int r; - input[0] = (char)0xF0; /* Literal length overflow */ - input[1] = (char)0xFF; - input[2] = (char)0xFF; - input[3] = (char)0xFF; - for(i = 4; i <= nbOf255+4; i++) input[i] = (char)0xff; - r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134); - if (r>0) goto _overflowError; - input[0] = (char)0x1F; /* Match length overflow */ - input[1] = (char)0x01; - input[2] = (char)0x01; - input[3] = (char)0x00; - r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134); - if (r>0) goto _overflowError; - - output = buffers[nbBuff-2]; /* Reverse in/out pointer order */ - input[0] = (char)0xF0; /* Literal length overflow */ - input[1] = (char)0xFF; - input[2] = (char)0xFF; - input[3] = (char)0xFF; - r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134); - if (r>0) goto _overflowError; - input[0] = (char)0x1F; /* Match length overflow */ - input[1] = (char)0x01; - input[2] = (char)0x01; - input[3] = (char)0x00; - r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134); - if (r>0) goto _overflowError; - } - } - - nbBuff++; -_endOfTests: - for (i=0 ; i<nbBuff; i++) free(buffers[i]); - if (!highAddress) DISPLAY("high address not possible \n"); - else DISPLAY("all overflows correctly detected \n"); - return 0; - -_overflowError: - DISPLAY("Address space overflow error !! \n"); - exit(1); -} - - -static void FUZ_displayUpdate(unsigned testNb) -{ - if ((FUZ_GetClockSpan(g_time) > g_refreshRate) | (g_displayLevel>=3)) { - g_time = clock(); - DISPLAY("\r%5u ", testNb); - if (g_displayLevel>=3) fflush(stdout); - } -} - - -/*! FUZ_findDiff() : -* find the first different byte between buff1 and buff2. -* presumes buff1 != buff2. -* presumes a difference exists before end of either buffer. -* Typically invoked after a checksum mismatch. -*/ -static void FUZ_findDiff(const void* buff1, const void* buff2) -{ - const BYTE* const b1 = (const BYTE*)buff1; - const BYTE* const b2 = (const BYTE*)buff2; - size_t i=0; - while (b1[i]==b2[i]) i++; - DISPLAY("Wrong Byte at position %u\n", (unsigned)i); -} - - -static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double compressibility, U32 duration_s) -{ - unsigned long long bytes = 0; - unsigned long long cbytes = 0; - unsigned long long hcbytes = 0; - unsigned long long ccbytes = 0; - void* CNBuffer; - char* compressedBuffer; - char* decodedBuffer; -# define FUZ_max LZ4_COMPRESSBOUND(LEN) - int ret; - unsigned cycleNb; -# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \ - printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; } -# define FUZ_DISPLAYTEST { testNb++; g_displayLevel<3 ? 0 : printf("%2u\b\b", testNb); if (g_displayLevel==4) fflush(stdout); } - void* stateLZ4 = malloc(LZ4_sizeofState()); - void* stateLZ4HC = malloc(LZ4_sizeofStateHC()); - LZ4_stream_t LZ4dict; - LZ4_streamHC_t LZ4dictHC; - U32 crcOrig, crcCheck; - U32 coreRandState = seed; - U32 randState = coreRandState ^ PRIME3; - int result = 0; - clock_t const clockStart = clock(); - clock_t const clockDuration = (clock_t)duration_s * CLOCKS_PER_SEC; - - - /* init */ - memset(&LZ4dict, 0, sizeof(LZ4dict)); - - /* Create compressible test buffer */ - CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); - FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); - compressedBuffer = (char*)malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE)); - decodedBuffer = (char*)malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE); - - /* move to startCycle */ - for (cycleNb = 0; cycleNb < startCycle; cycleNb++) { - (void)FUZ_rand(&coreRandState); - - if (0) { /* some problems can be related to dictionary re-use; in this case, enable this loop */ - int dictSize, blockSize, blockStart; - char* dict; - char* block; - FUZ_displayUpdate(cycleNb); - randState = coreRandState ^ PRIME3; - blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE; - blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); - dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; - if (dictSize > blockStart) dictSize = blockStart; - block = ((char*)CNBuffer) + blockStart; - dict = block - dictSize; - LZ4_loadDict(&LZ4dict, dict, dictSize); - LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); - LZ4_loadDict(&LZ4dict, dict, dictSize); - LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); - LZ4_loadDict(&LZ4dict, dict, dictSize); - LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); - } } - - /* Main test loop */ - for (cycleNb = startCycle; (cycleNb < nbCycles) || (FUZ_GetClockSpan(clockStart) < clockDuration) ; cycleNb++) { - U32 testNb = 0; - char* dict; - char* block; - int dictSize, blockSize, blockStart, compressedSize, HCcompressedSize; - int blockContinueCompressedSize; - - FUZ_displayUpdate(cycleNb); - (void)FUZ_rand(&coreRandState); - randState = coreRandState ^ PRIME3; - - /* Select block to test */ - blockSize = (FUZ_rand(&randState) % (FUZ_MAX_BLOCK_SIZE-1)) + 1; - blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); - dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; - if (dictSize > blockStart) dictSize = blockStart; - block = ((char*)CNBuffer) + blockStart; - dict = block - dictSize; - - /* Compression tests */ - - /* Test compression destSize */ - FUZ_DISPLAYTEST; - { int srcSize = blockSize; - int targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7; - char endCheck = FUZ_rand(&randState) & 255; - compressedBuffer[targetSize] = endCheck; - ret = LZ4_compress_destSize(block, compressedBuffer, &srcSize, targetSize); - FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_destSize() result larger than dst buffer !"); - FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_destSize() overwrite dst buffer !"); - FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_destSize() fed more than src buffer !"); - DISPLAYLEVEL(5, "destSize : %7i/%7i; content%7i/%7i ", ret, targetSize, srcSize, blockSize); - if (targetSize>0) { - FUZ_CHECKTEST((ret==0), "LZ4_compress_destSize() compression failed"); - /* check correctness */ - FUZ_DISPLAYTEST; - - crcOrig = XXH32(block, srcSize, 0); - compressedSize = ret; - endCheck = FUZ_rand(&randState) & 255; - decodedBuffer[srcSize] = endCheck; - ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compress_destSize"); - FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data"); - FUZ_CHECKTEST(decodedBuffer[srcSize] != endCheck, "LZ4_decompress_safe() overwrite dst buffer !"); - crcCheck = XXH32(decodedBuffer, srcSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe() corrupted decoded data"); - - DISPLAYLEVEL(5, " OK \n"); - } - else - DISPLAYLEVEL(5, " \n"); - } - - /* Test compression HC */ - FUZ_DISPLAYTEST; - ret = LZ4_compressHC(block, compressedBuffer, blockSize); - FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed"); - HCcompressedSize = ret; - - /* Test compression HC using external state */ - FUZ_DISPLAYTEST; - ret = LZ4_compressHC_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize); - FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed"); - - /* Test compression using external state */ - FUZ_DISPLAYTEST; - ret = LZ4_compress_withState(stateLZ4, block, compressedBuffer, blockSize); - FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed"); - - /* Test compression */ - FUZ_DISPLAYTEST; - ret = LZ4_compress(block, compressedBuffer, blockSize); - FUZ_CHECKTEST(ret==0, "LZ4_compress() failed"); - compressedSize = ret; - - /* Decompression tests */ - - crcOrig = XXH32(block, blockSize, 0); - - /* Test decoding with output size being exactly what's necessary => must work */ - FUZ_DISPLAYTEST; - ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space"); - FUZ_CHECKTEST(ret!=compressedSize, "LZ4_decompress_fast failed : did not fully read compressed data"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data"); - - /* Test decoding with one byte missing => must fail */ - FUZ_DISPLAYTEST; - decodedBuffer[blockSize-1] = 0; - ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small"); - FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer"); - - /* Test decoding with one byte too much => must fail */ - FUZ_DISPLAYTEST; - ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large"); - - /* Test decoding with output size exactly what's necessary => must work */ - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space"); - FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data"); - FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); - - // Test decoding with more than enough output size => must work - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - decodedBuffer[blockSize+1] = 0; - ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize+1); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite amply sufficient space"); - FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data"); - //FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe wrote more than (unknown) target size"); // well, is that an issue ? - FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ4_decompress_safe overrun specified output buffer size"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); - - // Test decoding with output size being one byte too short => must fail - FUZ_DISPLAYTEST; - decodedBuffer[blockSize-1] = 0; - ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-1); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short"); - FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe overrun specified output buffer size"); - - // Test decoding with output size being 10 bytes too short => must fail - FUZ_DISPLAYTEST; - if (blockSize>10) - { - decodedBuffer[blockSize-10] = 0; - ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-10); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being 10 bytes too short"); - FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe overrun specified output buffer size"); - } - - // Test decoding with input size being one byte too short => must fail - FUZ_DISPLAYTEST; - ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize); - - // Test decoding with input size being one byte too large => must fail - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize+1, blockSize); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large"); - FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size"); - - // Test partial decoding with target output size being max/2 => must work - FUZ_DISPLAYTEST; - ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize/2, blockSize); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space"); - - // Test partial decoding with target output size being just below max => must work - FUZ_DISPLAYTEST; - ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize-3, blockSize); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space"); - - /* Test Compression with limited output size */ - - /* Test compression with output size being exactly what's necessary (should work) */ - FUZ_DISPLAYTEST; - ret = LZ4_compress_limitedOutput(block, compressedBuffer, blockSize, compressedSize); - FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space"); - - /* Test compression with output size being exactly what's necessary and external state (should work) */ - FUZ_DISPLAYTEST; - ret = LZ4_compress_limitedOutput_withState(stateLZ4, block, compressedBuffer, blockSize, compressedSize); - FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput_withState() failed despite sufficient space"); - - /* Test HC compression with output size being exactly what's necessary (should work) */ - FUZ_DISPLAYTEST; - ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize); - FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space"); - - /* Test HC compression with output size being exactly what's necessary (should work) */ - FUZ_DISPLAYTEST; - ret = LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize); - FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space"); - - /* Test compression with missing bytes into output buffer => must fail */ - FUZ_DISPLAYTEST; - { int missingBytes = (FUZ_rand(&randState) % 0x3F) + 1; - if (missingBytes >= compressedSize) missingBytes = compressedSize-1; - missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ - compressedBuffer[compressedSize-missingBytes] = 0; - ret = LZ4_compress_limitedOutput(block, compressedBuffer, blockSize, compressedSize-missingBytes); - FUZ_CHECKTEST(ret, "LZ4_compress_limitedOutput should have failed (output buffer too small by %i byte)", missingBytes); - FUZ_CHECKTEST(compressedBuffer[compressedSize-missingBytes], "LZ4_compress_limitedOutput overran output buffer ! (%i missingBytes)", missingBytes) - } - - /* Test HC compression with missing bytes into output buffer => must fail */ - FUZ_DISPLAYTEST; - { int missingBytes = (FUZ_rand(&randState) % 0x3F) + 1; - if (missingBytes >= HCcompressedSize) missingBytes = HCcompressedSize-1; - missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ - compressedBuffer[HCcompressedSize-missingBytes] = 0; - ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes); - FUZ_CHECKTEST(ret, "LZ4_compressHC_limitedOutput should have failed (output buffer too small by %i byte)", missingBytes); - FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-missingBytes], "LZ4_compressHC_limitedOutput overran output buffer ! (%i missingBytes)", missingBytes) - } - - - /*-******************/ - /* Dictionary tests */ - /*-******************/ - - /* Compress using dictionary */ - FUZ_DISPLAYTEST; - { LZ4_stream_t LZ4_stream; - LZ4_resetStream(&LZ4_stream); - LZ4_compress_continue (&LZ4_stream, dict, compressedBuffer, dictSize); /* Just to fill hash tables */ - blockContinueCompressedSize = LZ4_compress_continue (&LZ4_stream, block, compressedBuffer, blockSize); - FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed"); - } - - /* Decompress with dictionary as prefix */ - FUZ_DISPLAYTEST; - memcpy(decodedBuffer, dict, dictSize); - ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer+dictSize, blockSize, decodedBuffer, dictSize); - FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_withPrefix64k did not read all compressed block input"); - crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); - if (crcCheck!=crcOrig) { - int i=0; - while (block[i]==decodedBuffer[i]) i++; - printf("Wrong Byte at position %i/%i\n", i, blockSize); - - } - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_withPrefix64k corrupted decoded data (dict %i)", dictSize); - - FUZ_DISPLAYTEST; - ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize); - FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); - crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); - - /* Compress using External dictionary */ - FUZ_DISPLAYTEST; - dict -= (FUZ_rand(&randState) & 0xF) + 1; /* Separation, so it is an ExtDict */ - if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; - LZ4_loadDict(&LZ4dict, dict, dictSize); - blockContinueCompressedSize = LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); - FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed"); - - FUZ_DISPLAYTEST; - LZ4_loadDict(&LZ4dict, dict, dictSize); - ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); - FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize); - - FUZ_DISPLAYTEST; - LZ4_loadDict(&LZ4dict, dict, dictSize); - ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize); - FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); - FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer"); - - /* Decompress with dictionary as external */ - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize); - FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input"); - FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size") - crcCheck = XXH32(decodedBuffer, blockSize, 0); - if (crcCheck!=crcOrig) - FUZ_findDiff(block, decodedBuffer); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); - - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); - FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); - FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size") - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); - - FUZ_DISPLAYTEST; - decodedBuffer[blockSize-1] = 0; - ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)"); - FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size"); - - FUZ_DISPLAYTEST; - decodedBuffer[blockSize-1] = 0; - ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)"); - FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); - - FUZ_DISPLAYTEST; - { U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2; - if ((U32)blockSize > missingBytes) { - decodedBuffer[blockSize-missingBytes] = 0; - ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-missingBytes, dict, dictSize); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%u byte)", missingBytes); - FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize); - } } - - /* Compress HC using External dictionary */ - FUZ_DISPLAYTEST; - dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */ - if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; - LZ4_resetStreamHC (&LZ4dictHC, FUZ_rand(&randState) & 0x7); - LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); - blockContinueCompressedSize = LZ4_compressHC_continue(&LZ4dictHC, block, compressedBuffer, blockSize); - FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compressHC_continue failed"); - - FUZ_DISPLAYTEST; - LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); - ret = LZ4_compressHC_limitedOutput_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); - FUZ_CHECKTEST(ret>0, "LZ4_compressHC_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer"); - - FUZ_DISPLAYTEST; - LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); - ret = LZ4_compressHC_limitedOutput_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize); - FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); - FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer"); - - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); - FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); - FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size") - crcCheck = XXH32(decodedBuffer, blockSize, 0); - if (crcCheck!=crcOrig) - FUZ_findDiff(block, decodedBuffer); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); - - /* ***** End of tests *** */ - /* Fill stats */ - bytes += blockSize; - cbytes += compressedSize; - hcbytes += HCcompressedSize; - ccbytes += blockContinueCompressedSize; - } - - if (nbCycles<=1) nbCycles = cycleNb; /* end by time */ - bytes += !bytes; /* avoid division by 0 */ - printf("\r%7u /%7u - ", cycleNb, nbCycles); - printf("all tests completed successfully \n"); - printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100); - printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100); - printf("ratio with dict: %0.3f%%\n", (double)ccbytes/bytes*100); - - /* release memory */ - { -_exit: - free(CNBuffer); - free(compressedBuffer); - free(decodedBuffer); - free(stateLZ4); - free(stateLZ4HC); - return result; - -_output_error: - result = 1; - goto _exit; - } -} - - -#define testInputSize (192 KB) -#define testCompressedSize (128 KB) -#define ringBufferSize (8 KB) - -static void FUZ_unitTests(void) -{ - const unsigned testNb = 0; - const unsigned seed = 0; - const unsigned cycleNb= 0; - char testInput[testInputSize]; - char testCompressed[testCompressedSize]; - char testVerify[testInputSize]; - char ringBuffer[ringBufferSize]; - U32 randState = 1; - - /* Init */ - FUZ_fillCompressibleNoiseBuffer(testInput, testInputSize, 0.50, &randState); - - /* 32-bits address space overflow test */ - FUZ_AddressOverflow(); - - /* LZ4 streaming tests */ - { LZ4_stream_t* statePtr; - LZ4_stream_t streamingState; - U64 crcOrig; - U64 crcNew; - int result; - - /* Allocation test */ - statePtr = LZ4_createStream(); - FUZ_CHECKTEST(statePtr==NULL, "LZ4_createStream() allocation failed"); - LZ4_freeStream(statePtr); - - /* simple compression test */ - crcOrig = XXH64(testInput, testCompressedSize, 0); - LZ4_resetStream(&streamingState); - result = LZ4_compress_limitedOutput_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "LZ4_compress_limitedOutput_continue() compression failed"); - - result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); - crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); - - /* ring buffer test */ - { XXH64_state_t xxhOrig; - XXH64_state_t xxhNew; - LZ4_streamDecode_t decodeState; - const U32 maxMessageSizeLog = 10; - const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1; - U32 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1; - U32 iNext = 0; - U32 rNext = 0; - U32 dNext = 0; - const U32 dBufferSize = ringBufferSize + maxMessageSizeMask; - - XXH64_reset(&xxhOrig, 0); - XXH64_reset(&xxhNew, 0); - LZ4_resetStream(&streamingState); - LZ4_setStreamDecode(&decodeState, NULL, 0); - - while (iNext + messageSize < testCompressedSize) { - XXH64_update(&xxhOrig, testInput + iNext, messageSize); - crcOrig = XXH64_digest(&xxhOrig); - - memcpy (ringBuffer + rNext, testInput + iNext, messageSize); - result = LZ4_compress_limitedOutput_continue(&streamingState, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); - FUZ_CHECKTEST(result==0, "LZ4_compress_limitedOutput_continue() compression failed"); - - result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize); - FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed"); - - XXH64_update(&xxhNew, testVerify + dNext, messageSize); - crcNew = XXH64_digest(&xxhNew); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); - - /* prepare next message */ - iNext += messageSize; - rNext += messageSize; - dNext += messageSize; - messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1; - if (rNext + messageSize > ringBufferSize) rNext = 0; - if (dNext + messageSize > dBufferSize) dNext = 0; - } - } - } - - /* LZ4 HC streaming tests */ - { LZ4_streamHC_t* sp; - LZ4_streamHC_t sHC; - U64 crcOrig; - U64 crcNew; - int result; - - /* Allocation test */ - sp = LZ4_createStreamHC(); - FUZ_CHECKTEST(sp==NULL, "LZ4_createStreamHC() allocation failed"); - LZ4_freeStreamHC(sp); - - /* simple HC compression test */ - crcOrig = XXH64(testInput, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, 0); - result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); - - result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); - crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); - - /* simple dictionary HC compression test */ - crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, 0); - LZ4_loadDictHC(&sHC, testInput, 64 KB); - result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); - - result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 64 KB); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed"); - crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() simple dictionary decompression test : corruption"); - - /* multiple HC compression test with dictionary */ - { int result1, result2; - int segSize = testCompressedSize / 2; - crcOrig = XXH64(testInput + segSize, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, 0); - LZ4_loadDictHC(&sHC, testInput, segSize); - result1 = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1); - FUZ_CHECKTEST(result1==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result1); - result2 = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + 2*segSize, testCompressed+result1, segSize, segSize-1); - FUZ_CHECKTEST(result2==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result2); - - result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result1, segSize, testInput, segSize); - FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 1 failed"); - result = LZ4_decompress_safe_usingDict(testCompressed+result1, testVerify+segSize, result2, segSize, testInput, 2*segSize); - FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 2 failed"); - crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); - } - - /* remote dictionary HC compression test */ - crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, 0); - LZ4_loadDictHC(&sHC, testInput, 32 KB); - result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result); - - result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 32 KB); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe_usingDict() decompression failed following remote dictionary HC compression test"); - crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() decompression corruption"); - - /* multiple HC compression with ext. dictionary */ - { XXH64_state_t crcOrigState; - XXH64_state_t crcNewState; - const char* dict = testInput + 3; - int dictSize = (FUZ_rand(&randState) & 8191); - char* dst = testVerify; - - size_t segStart = dictSize + 7; - int segSize = (FUZ_rand(&randState) & 8191); - int segNb = 1; - - LZ4_resetStreamHC(&sHC, 0); - LZ4_loadDictHC(&sHC, dict, dictSize); - - XXH64_reset(&crcOrigState, 0); - XXH64_reset(&crcNewState, 0); - - while (segStart + segSize < testInputSize) { - XXH64_update(&crcOrigState, testInput + segStart, segSize); - crcOrig = XXH64_digest(&crcOrigState); - result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + segStart, testCompressed, segSize, LZ4_compressBound(segSize)); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); - - result = LZ4_decompress_safe_usingDict(testCompressed, dst, result, segSize, dict, dictSize); - FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %i failed", segNb); - XXH64_update(&crcNewState, dst, segSize); - crcNew = XXH64_digest(&crcNewState); - if (crcOrig!=crcNew) { - size_t c=0; - while (dst[c] == testInput[segStart+c]) c++; - DISPLAY("Bad decompression at %u / %u \n", (U32)c, (U32)segSize); - } - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() part %i corruption", segNb); - - dict = dst; - //dict = testInput + segStart; - dictSize = segSize; - - dst += segSize + 1; - segNb ++; - - segStart += segSize + (FUZ_rand(&randState) & 0xF) + 1; - segSize = (FUZ_rand(&randState) & 8191); - } - } - - /* ring buffer test */ - { XXH64_state_t xxhOrig; - XXH64_state_t xxhNew; - LZ4_streamDecode_t decodeState; - const U32 maxMessageSizeLog = 10; - const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1; - U32 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1; - U32 iNext = 0; - U32 rNext = 0; - U32 dNext = 0; - const U32 dBufferSize = ringBufferSize + maxMessageSizeMask; - - XXH64_reset(&xxhOrig, 0); - XXH64_reset(&xxhNew, 0); - LZ4_resetStreamHC(&sHC, 0); - LZ4_setStreamDecode(&decodeState, NULL, 0); - - while (iNext + messageSize < testCompressedSize) { - XXH64_update(&xxhOrig, testInput + iNext, messageSize); - crcOrig = XXH64_digest(&xxhOrig); - - memcpy (ringBuffer + rNext, testInput + iNext, messageSize); - result = LZ4_compressHC_limitedOutput_continue(&sHC, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); - - result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize); - FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed"); - - XXH64_update(&xxhNew, testVerify + dNext, messageSize); - crcNew = XXH64_digest(&xxhNew); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); - - /* prepare next message */ - iNext += messageSize; - rNext += messageSize; - dNext += messageSize; - messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1; - if (rNext + messageSize > ringBufferSize) rNext = 0; - if (dNext + messageSize > dBufferSize) dNext = 0; - } - } - - /* small decoder-side ring buffer test */ - { XXH64_state_t xxhOrig; - XXH64_state_t xxhNew; - LZ4_streamDecode_t decodeState; - const U32 maxMessageSizeLog = 12; - const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1; - U32 messageSize; - U32 totalMessageSize = 0; - U32 iNext = 0; - U32 dNext = 0; - const U32 dBufferSize = 64 KB; - - XXH64_reset(&xxhOrig, 0); - XXH64_reset(&xxhNew, 0); - LZ4_resetStreamHC(&sHC, 0); - LZ4_setStreamDecode(&decodeState, NULL, 0); - -#define BSIZE1 65537 -#define BSIZE2 16435 - - /* first block */ - - messageSize = BSIZE1; - XXH64_update(&xxhOrig, testInput + iNext, messageSize); - crcOrig = XXH64_digest(&xxhOrig); - - result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); - - result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize); - FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed"); - - XXH64_update(&xxhNew, testVerify + dNext, messageSize); - crcNew = XXH64_digest(&xxhNew); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); - - /* prepare next message */ - dNext += messageSize; - totalMessageSize += messageSize; - messageSize = BSIZE2; - iNext = 132000; - memcpy(testInput + iNext, testInput + 8, messageSize); - if (dNext > dBufferSize) dNext = 0; - - while (totalMessageSize < 9 MB) { - XXH64_update(&xxhOrig, testInput + iNext, messageSize); - crcOrig = XXH64_digest(&xxhOrig); - - result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); - - result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize); - FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed"); - - XXH64_update(&xxhNew, testVerify + dNext, messageSize); - crcNew = XXH64_digest(&xxhNew); - if (crcOrig != crcNew) - FUZ_findDiff(testInput + iNext, testVerify + dNext); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption during small decoder-side ring buffer test"); - - /* prepare next message */ - dNext += messageSize; - totalMessageSize += messageSize; - messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1; - iNext = (FUZ_rand(&randState) & 65535); - if (dNext > dBufferSize) dNext = 0; - } - } - } - - printf("All unit tests completed successfully \n"); - return; -_output_error: - exit(1); -} - - -static int FUZ_usage(char* programName) -{ - DISPLAY( "Usage :\n"); - DISPLAY( " %s [args]\n", programName); - DISPLAY( "\n"); - DISPLAY( "Arguments :\n"); - DISPLAY( " -i# : Nb of tests (default:%i) \n", NB_ATTEMPTS); - DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n"); - DISPLAY( " -s# : Select seed (default:prompt user)\n"); - DISPLAY( " -t# : Select starting test number (default:0)\n"); - DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); - DISPLAY( " -v : verbose\n"); - DISPLAY( " -p : pause at the end\n"); - DISPLAY( " -h : display help and exit\n"); - return 0; -} - - -int main(int argc, char** argv) -{ - U32 seed=0; - int seedset=0; - int argNb; - int nbTests = NB_ATTEMPTS; - int testNb = 0; - int proba = FUZ_COMPRESSIBILITY_DEFAULT; - int pause = 0; - char* programName = argv[0]; - U32 duration = 0; - - /* Check command line */ - for(argNb=1; argNb<argc; argNb++) { - char* argument = argv[argNb]; - - if(!argument) continue; // Protection if argument empty - - // Decode command (note : aggregated commands are allowed) - if (argument[0]=='-') { - if (!strcmp(argument, "--no-prompt")) { pause=0; seedset=1; g_displayLevel=1; continue; } - argument++; - - while (*argument!=0) { - switch(*argument) - { - case 'h': /* display help */ - return FUZ_usage(programName); - - case 'v': /* verbose mode */ - argument++; - g_displayLevel=4; - break; - - case 'p': /* pause at the end */ - argument++; - pause=1; - break; - - case 'i': - argument++; - nbTests = 0; duration = 0; - while ((*argument>='0') && (*argument<='9')) { - nbTests *= 10; - nbTests += *argument - '0'; - argument++; - } - break; - - case 'T': - argument++; - nbTests = 0; duration = 0; - for (;;) { - switch(*argument) - { - case 'm': duration *= 60; argument++; continue; - case 's': - case 'n': argument++; continue; - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': duration *= 10; duration += *argument++ - '0'; continue; - } - break; - } - break; - - case 's': - argument++; - seed=0; seedset=1; - while ((*argument>='0') && (*argument<='9')) { - seed *= 10; - seed += *argument - '0'; - argument++; - } - break; - - case 't': /* select starting test nb */ - argument++; - testNb=0; - while ((*argument>='0') && (*argument<='9')) { - testNb *= 10; - testNb += *argument - '0'; - argument++; - } - break; - - case 'P': /* change probability */ - argument++; - proba=0; - while ((*argument>='0') && (*argument<='9')) { - proba *= 10; - proba += *argument - '0'; - argument++; - } - if (proba<0) proba=0; - if (proba>100) proba=100; - break; - default: ; - } - } - } - } - - printf("Starting LZ4 fuzzer (%i-bits, v%s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION_STRING); - - if (!seedset) { - time_t const t = time(NULL); - U32 const h = XXH32(&t, sizeof(t), 1); - seed = h % 10000; - } - printf("Seed = %u\n", seed); - - if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba); - - if ((seedset==0) && (testNb==0)) FUZ_unitTests(); - - if (nbTests<=0) nbTests=1; - - { int const result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100, duration); - if (pause) { - DISPLAY("press enter ... \n"); - (void)getchar(); - } - return result; - } -} |