diff options
author | Yann Collet <Cyan4973@users.noreply.github.com> | 2019-04-16 17:07:41 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-04-16 17:07:41 (GMT) |
commit | f1226ac53dee1b41a801e8003bb3708a2e671d12 (patch) | |
tree | dd17db72ce5d38e4d1cc65e4a298b11d2d96c817 /tests | |
parent | 01d2a721d393646384291af2f6f2f940493cd78f (diff) | |
parent | fc3176f6aa6b8034e0a27598c23bdda559b5cf9c (diff) | |
download | lz4-f1226ac53dee1b41a801e8003bb3708a2e671d12.zip lz4-f1226ac53dee1b41a801e8003bb3708a2e671d12.tar.gz lz4-f1226ac53dee1b41a801e8003bb3708a2e671d12.tar.bz2 |
Merge pull request #670 from lz4/dev
v1.9.0
Diffstat (limited to 'tests')
-rw-r--r-- | tests/.gitignore | 1 | ||||
-rw-r--r-- | tests/Makefile | 82 | ||||
-rw-r--r-- | tests/checkFrame.c | 311 | ||||
-rw-r--r-- | tests/frametest.c | 137 | ||||
-rw-r--r-- | tests/fullbench.c | 28 | ||||
-rw-r--r-- | tests/fuzzer.c | 267 | ||||
-rwxr-xr-x | tests/test_custom_block_sizes.sh | 72 |
7 files changed, 725 insertions, 173 deletions
diff --git a/tests/.gitignore b/tests/.gitignore index 9aa42a0..c4f9092 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -10,6 +10,7 @@ fuzzer32 fasttest roundTripTest checkTag +checkFrame # test artefacts tmp* diff --git a/tests/Makefile b/tests/Makefile index 3de111b..70cae63 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -63,7 +63,7 @@ NB_LOOPS ?= -i1 default: all -all: fullbench fuzzer frametest roundTripTest datagen +all: fullbench fuzzer frametest roundTripTest datagen checkFrame all32: CFLAGS+=-m32 all32: all @@ -109,17 +109,21 @@ roundTripTest : lz4.o lz4hc.o xxhash.o roundTripTest.c datagen : $(PRGDIR)/datagen.c datagencli.c $(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT) +checkFrame : lz4frame.o lz4.o lz4hc.o xxhash.o checkFrame.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + clean: @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) @$(MAKE) -C $(PRGDIR) $@ > $(VOID) - @$(RM) core *.o *.test tmp* \ + @$(RM) -rf core *.o *.test tmp* \ fullbench-dll$(EXT) fullbench-lib$(EXT) \ fullbench$(EXT) fullbench32$(EXT) \ fuzzer$(EXT) fuzzer32$(EXT) \ frametest$(EXT) frametest32$(EXT) \ fasttest$(EXT) roundTripTest$(EXT) \ - datagen$(EXT) checkTag$(EXT) - @rm -fR $(TESTDIR) + datagen$(EXT) checkTag$(EXT) \ + frameTest$(EXT) + @$(RM) -rf $(TESTDIR) @echo Cleaning completed .PHONY: versionsTest @@ -147,12 +151,25 @@ endif DD:=dd +.PHONY: list +list: + @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs -test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-install +.PHONY: test +test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-install test-amalgamation +.PHONY: test32 test32: CFLAGS+=-m32 test32: test +.PHONY: test-amalgamation +test-amalgamation: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c + cat $(LZ4DIR)/lz4.c > lz4_all.c + cat $(LZ4DIR)/lz4hc.c >> lz4_all.c + cat $(LZ4DIR)/lz4frame.c >> lz4_all.c + $(CC) -I$(LZ4DIR) -c lz4_all.c + $(RM) lz4_all.c + test-install: lz4 lib liblz4.pc lz4_root=.. ./test_install.sh @@ -205,7 +222,7 @@ test-lz4-frame-concatenation: lz4 datagen @$(LZ4) -zq tmp-lfc-nonempty > tmp-lfc-nonempty.lz4 cat tmp-lfc-nonempty.lz4 tmp-lfc-empty.lz4 tmp-lfc-nonempty.lz4 > tmp-lfc-concat.lz4 $(LZ4) -d tmp-lfc-concat.lz4 > tmp-lfc-result - sdiff tmp-lfc-src tmp-lfc-result + cmp tmp-lfc-src tmp-lfc-result @$(RM) tmp-lfc-* @echo frame concatenation test completed @@ -213,13 +230,36 @@ test-lz4-multiple: lz4 datagen @echo "\n ---- test multiple files ----" @./datagen -s1 > tmp-tlm1 2> $(VOID) @./datagen -s2 -g100K > tmp-tlm2 2> $(VOID) - @./datagen -s3 -g1M > tmp-tlm3 2> $(VOID) + @./datagen -s3 -g200K > tmp-tlm3 2> $(VOID) + # compress multiple files : one .lz4 per source file $(LZ4) -f -m tmp-tlm* - ls -ls tmp-tlm* - @$(RM) tmp-tlm1 tmp-tlm2 tmp-tlm3 - $(LZ4) -df -m tmp-tlm*.lz4 - ls -ls tmp-tlm* - $(LZ4) -f -m tmp-tlm1 notHere tmp-tlm2; echo $$? + test -f tmp-tlm1.lz4 + test -f tmp-tlm2.lz4 + test -f tmp-tlm3.lz4 + # decompress multiple files : one output file per .lz4 + mv tmp-tlm1 tmp-tlm1-orig + mv tmp-tlm2 tmp-tlm2-orig + mv tmp-tlm3 tmp-tlm3-orig + $(LZ4) -d -f -m tmp-tlm*.lz4 + cmp tmp-tlm1 tmp-tlm1-orig # must be identical + cmp tmp-tlm2 tmp-tlm2-orig + cmp tmp-tlm3 tmp-tlm3-orig + # compress multiple files into stdout + cat tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 > tmp-tlm-concat1 + $(RM) *.lz4 + $(LZ4) -m tmp-tlm1 tmp-tlm2 tmp-tlm3 -c > tmp-tlm-concat2 + test ! -f tmp-tlm1.lz4 # must not create .lz4 artefact + cmp tmp-tlm-concat1 tmp-tlm-concat2 # must be equivalent + # decompress multiple files into stdout + $(RM) tmp-tlm-concat1 tmp-tlm-concat2 + $(LZ4) -f -m tmp-tlm1 tmp-tlm2 tmp-tlm3 # generate .lz4 to decompress + cat tmp-tlm1 tmp-tlm2 tmp-tlm3 > tmp-tlm-concat1 # create concatenated reference + $(RM) tmp-tlm1 tmp-tlm2 tmp-tlm3 + $(LZ4) -d -m tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 -c > tmp-tlm-concat2 + test ! -f tmp-tlm1 # must not create file artefact + cmp tmp-tlm-concat1 tmp-tlm-concat2 # must be equivalent + # compress multiple files, one of which is absent (must fail) + ! $(LZ4) -f -m tmp-tlm-concat1 notHere tmp-tlm-concat2 # must fail : notHere not present @$(RM) tmp-tlm* test-lz4-basic: lz4 datagen unlz4 lz4cat @@ -231,7 +271,7 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat $(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec $(LZ4) --no-frame-crc < tmp-tlb-dg20k | $(LZ4) -d > tmp-tlb-dec $(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec - ./datagen | $(LZ4) | $(LZ4) -t + ./datagen | $(LZ4) -BI | $(LZ4) -t ./datagen -g6M -P99 | $(LZ4) -9BD | $(LZ4) -t ./datagen -g17M | $(LZ4) -9v | $(LZ4) -qt ./datagen -g33M | $(LZ4) --no-frame-crc | $(LZ4) -t @@ -273,6 +313,11 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat test "$(shell ./datagen -g20KB | $(LZ4) -c --fast=1 | wc -c)" -eq "$(shell ./datagen -g20KB| $(LZ4) -c --fast| wc -c)" # checks default fast compression is -1 ! $(LZ4) -c --fast=0 tmp-tlb-dg20K # lz4 should fail when fast=0 ! $(LZ4) -c --fast=-1 tmp-tlb-dg20K # lz4 should fail when fast=-1 + # Test for #596 + @echo "TEST" > tmp-tlb-test + $(LZ4) tmp-tlb-test + $(LZ4) tmp-tlb-test.lz4 tmp-tlb-test2 + $(DIFF) -q tmp-tlb-test tmp-tlb-test2 @$(RM) tmp-tlb* @@ -305,7 +350,7 @@ test-lz4-dict: lz4 datagen test-lz4-hugefile: lz4 datagen @echo "\n ---- test huge files compression/decompression ----" ./datagen -g6GB | $(LZ4) -vB5D | $(LZ4) -qt - ./datagen -g6GB | $(LZ4) -v5BD | $(LZ4) -qt + ./datagen -g5GB | $(LZ4) -v4BD | $(LZ4) -qt # test large file size [2-4] GB @./datagen -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmphf1 @ls -ls tmphf1 @@ -321,8 +366,11 @@ test-lz4-testmode: lz4 datagen ! ./datagen | $(LZ4) -t ! ./datagen | $(LZ4) -tf @echo "\n ---- pass-through mode ----" - ! ./datagen | $(LZ4) -d > $(VOID) - ./datagen | $(LZ4) -df > $(VOID) + @echo "Why hello there " > tmp-tlt2.lz4 + ! $(LZ4) -f tmp-tlt2.lz4 > $(VOID) + ! ./datagen | $(LZ4) -dc > $(VOID) + ! ./datagen | $(LZ4) -df > $(VOID) + ./datagen | $(LZ4) -dcf > $(VOID) @echo "Hello World !" > tmp-tlt1 $(LZ4) -dcf tmp-tlt1 @echo "from underground..." > tmp-tlt2 @@ -331,7 +379,7 @@ test-lz4-testmode: lz4 datagen ! $(LZ4) file-does-not-exist ! $(LZ4) -f file-does-not-exist ! $(LZ4) -fm file1-dne file2-dne - @$(RM) tmp-tlt + @$(RM) tmp-tlt tmp-tlt1 tmp-tlt2 tmp-tlt2.lz4 test-lz4-opt-parser: lz4 datagen @echo "\n ---- test opt-parser ----" diff --git a/tests/checkFrame.c b/tests/checkFrame.c new file mode 100644 index 0000000..50c0405 --- /dev/null +++ b/tests/checkFrame.c @@ -0,0 +1,311 @@ + /* + checkFrame - verify frame headers + Copyright (C) Yann Collet 2014-present + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - LZ4 homepage : http://www.lz4.org + - LZ4 source repository : https://github.com/lz4/lz4 + */ + + /*-************************************ + * Compiler specific + **************************************/ + #ifdef _MSC_VER /* Visual Studio */ + # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ + # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ + #endif + + + /*-************************************ + * Includes + **************************************/ + #include "util.h" /* U32 */ + #include <stdlib.h> /* malloc, free */ + #include <stdio.h> /* fprintf */ + #include <string.h> /* strcmp */ + #include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */ + #include <assert.h> + #include "lz4frame.h" /* include multiple times to test correctness/safety */ + #include "lz4frame.h" + #define LZ4F_STATIC_LINKING_ONLY + #include "lz4frame.h" + #include "lz4frame.h" + #include "lz4.h" /* LZ4_VERSION_STRING */ + #define XXH_STATIC_LINKING_ONLY + #include "xxhash.h" /* XXH64 */ + + + /*-************************************ + * Constants + **************************************/ + #define KB *(1U<<10) + #define MB *(1U<<20) + #define GB *(1U<<30) + + + /*-************************************ + * Macros + **************************************/ + #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) + #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } + + /************************************** + * Exceptions + ***************************************/ + #ifndef DEBUG + # define DEBUG 0 + #endif + #define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); + #define EXM_THROW(error, ...) \ +{ \ + DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAYLEVEL(1, "Error %i : ", error); \ + DISPLAYLEVEL(1, __VA_ARGS__); \ + DISPLAYLEVEL(1, " \n"); \ + return(error); \ +} + + + +/*-*************************************** +* Local Parameters +*****************************************/ +static U32 no_prompt = 0; +static U32 displayLevel = 2; +static U32 use_pause = 0; + + +/*-******************************************************* +* Fuzzer functions +*********************************************************/ +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +#define MAX(a,b) ( (a) > (b) ? (a) : (b) ) + +typedef struct { + void* srcBuffer; + size_t srcBufferSize; + void* dstBuffer; + size_t dstBufferSize; + LZ4F_decompressionContext_t ctx; +} cRess_t; + +static int createCResources(cRess_t *ress) +{ + ress->srcBufferSize = 4 MB; + ress->srcBuffer = malloc(ress->srcBufferSize); + ress->dstBufferSize = 4 MB; + ress->dstBuffer = malloc(ress->dstBufferSize); + + if (!ress->srcBuffer || !ress->dstBuffer) { + free(ress->srcBuffer); + free(ress->dstBuffer); + EXM_THROW(20, "Allocation error : not enough memory"); + } + + if (LZ4F_isError( LZ4F_createDecompressionContext(&(ress->ctx), LZ4F_VERSION) )) { + free(ress->srcBuffer); + free(ress->dstBuffer); + EXM_THROW(21, "Unable to create decompression context"); + } + return 0; +} + +static void freeCResources(cRess_t ress) +{ + free(ress.srcBuffer); + free(ress.dstBuffer); + + (void) LZ4F_freeDecompressionContext(ress.ctx); +} + +int frameCheck(cRess_t ress, FILE* const srcFile, unsigned bsid, size_t blockSize) +{ + LZ4F_errorCode_t nextToLoad = 0; + size_t curblocksize = 0; + int partialBlock = 0; + + /* Main Loop */ + for (;;) { + size_t readSize; + size_t pos = 0; + size_t decodedBytes = ress.dstBufferSize; + size_t remaining; + LZ4F_frameInfo_t frameInfo; + + /* Read input */ + readSize = fread(ress.srcBuffer, 1, ress.srcBufferSize, srcFile); + if (!readSize) break; /* reached end of file or stream */ + + while (pos < readSize) { /* still to read */ + /* Decode Input (at least partially) */ + if (!nextToLoad) { + /* LZ4F_decompress returned 0 : starting new frame */ + curblocksize = 0; + remaining = readSize - pos; + nextToLoad = LZ4F_getFrameInfo(ress.ctx, &frameInfo, (char*)(ress.srcBuffer)+pos, &remaining); + if (LZ4F_isError(nextToLoad)) + EXM_THROW(22, "Error getting frame info: %s", + LZ4F_getErrorName(nextToLoad)); + if (frameInfo.blockSizeID != bsid) + EXM_THROW(23, "Block size ID %u != expected %u", + frameInfo.blockSizeID, bsid); + pos += remaining; + /* nextToLoad should be block header size */ + remaining = nextToLoad; + decodedBytes = ress.dstBufferSize; + nextToLoad = LZ4F_decompress(ress.ctx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, NULL); + if (LZ4F_isError(nextToLoad)) EXM_THROW(24, "Decompression error : %s", LZ4F_getErrorName(nextToLoad)); + pos += remaining; + } + decodedBytes = ress.dstBufferSize; + /* nextToLoad should be just enough to cover the next block */ + if (nextToLoad > (readSize - pos)) { + /* block is not fully contained in current buffer */ + partialBlock = 1; + remaining = readSize - pos; + } else { + if (partialBlock) { + partialBlock = 0; + } + remaining = nextToLoad; + } + nextToLoad = LZ4F_decompress(ress.ctx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, NULL); + if (LZ4F_isError(nextToLoad)) EXM_THROW(24, "Decompression error : %s", LZ4F_getErrorName(nextToLoad)); + curblocksize += decodedBytes; + pos += remaining; + if (!partialBlock) { + /* detect small block due to end of frame; the final 4-byte frame checksum could be left in the buffer */ + if ((curblocksize != 0) && (nextToLoad > 4)) { + if (curblocksize != blockSize) + EXM_THROW(25, "Block size %u != expected %u, pos %u\n", + (unsigned)curblocksize, (unsigned)blockSize, (unsigned)pos); + } + curblocksize = 0; + } + } + } + /* can be out because readSize == 0, which could be an fread() error */ + if (ferror(srcFile)) EXM_THROW(26, "Read error"); + + if (nextToLoad!=0) EXM_THROW(27, "Unfinished stream"); + + return 0; +} + +int FUZ_usage(const char* programName) +{ + DISPLAY( "Usage :\n"); + DISPLAY( " %s [args] filename\n", programName); + DISPLAY( "\n"); + DISPLAY( "Arguments :\n"); + DISPLAY( " -b# : expected blocksizeID [4-7] (required)\n"); + DISPLAY( " -B# : expected blocksize [32-4194304] (required)\n"); + DISPLAY( " -v : verbose\n"); + DISPLAY( " -h : display help and exit\n"); + return 0; +} + + +int main(int argc, const char** argv) +{ + int argNb; + unsigned bsid=0; + size_t blockSize=0; + const char* const programName = argv[0]; + + /* Check command line */ + for (argNb=1; argNb<argc; argNb++) { + const char* argument = argv[argNb]; + + if(!argument) continue; /* Protection if argument empty */ + + /* Decode command (note : aggregated short commands are allowed) */ + if (argument[0]=='-') { + if (!strcmp(argument, "--no-prompt")) { + no_prompt=1; + displayLevel=1; + continue; + } + argument++; + + while (*argument!=0) { + switch(*argument) + { + case 'h': + return FUZ_usage(programName); + case 'v': + argument++; + displayLevel++; + break; + case 'q': + argument++; + displayLevel--; + break; + case 'p': /* pause at the end */ + argument++; + use_pause = 1; + break; + + case 'b': + argument++; + bsid=0; + while ((*argument>='0') && (*argument<='9')) { + bsid *= 10; + bsid += (unsigned)(*argument - '0'); + argument++; + } + break; + + case 'B': + argument++; + blockSize=0; + while ((*argument>='0') && (*argument<='9')) { + blockSize *= 10; + blockSize += (size_t)(*argument - '0'); + argument++; + } + break; + + default: + ; + return FUZ_usage(programName); + } + } + } else { + int err; + FILE *srcFile; + cRess_t ress; + if (bsid == 0 || blockSize == 0) + return FUZ_usage(programName); + DISPLAY("Starting frame checker (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION_STRING); + err = createCResources(&ress); + if (err) return (err); + srcFile = fopen(argument, "rb"); + if ( srcFile==NULL ) { + freeCResources(ress); + EXM_THROW(1, "%s: %s \n", argument, strerror(errno)); + } + err = frameCheck(ress, srcFile, bsid, blockSize); + freeCResources(ress); + fclose(srcFile); + return (err); + } + } + return 0; +} diff --git a/tests/frametest.c b/tests/frametest.c index 4efeb6f..a5197ff 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -27,8 +27,8 @@ * Compiler specific **************************************/ #ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ #endif @@ -55,7 +55,7 @@ static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32) { BYTE* dstPtr = (BYTE*)dstVoidPtr; - dstPtr[0] = (BYTE)value32; + dstPtr[0] = (BYTE) value32; dstPtr[1] = (BYTE)(value32 >> 8); dstPtr[2] = (BYTE)(value32 >> 16); dstPtr[3] = (BYTE)(value32 >> 24); @@ -77,7 +77,6 @@ static const U32 prime1 = 2654435761U; static const U32 prime2 = 2246822519U; - /*-************************************ * Macros **************************************/ @@ -160,7 +159,7 @@ static unsigned FUZ_highbit(U32 v32) { unsigned nbBits = 0; if (v32==0) return 0; - while (v32) v32 >>= 1, nbBits ++; + while (v32) {v32 >>= 1; nbBits ++;} return nbBits; } @@ -211,9 +210,13 @@ int basicTests(U32 seed, double compressibility) CHECK ( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) ); DISPLAYLEVEL(3, "LZ4F_getFrameInfo on null-content frame (#157) \n"); - { size_t avail_in = cSize; - LZ4F_frameInfo_t frame_info; + assert(cSize >= LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH); + { LZ4F_frameInfo_t frame_info; + size_t const fhs = LZ4F_headerSize(compressedBuffer, LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH); + size_t avail_in = fhs; + CHECK( fhs ); CHECK( LZ4F_getFrameInfo(dCtx, &frame_info, compressedBuffer, &avail_in) ); + if (avail_in != fhs) goto _output_error; /* must consume all, since header size is supposed to be exact */ } DISPLAYLEVEL(3, "LZ4F_freeDecompressionContext \n"); @@ -270,7 +273,7 @@ int basicTests(U32 seed, double compressibility) if (decResult != 0) goto _output_error; /* should finish now */ op += oSize; if (op>oend) { DISPLAY("decompression write overflow \n"); goto _output_error; } - { U64 const crcDest = XXH64(decodedBuffer, op-ostart, 1); + { U64 const crcDest = XXH64(decodedBuffer, (size_t)(op-ostart), 1); if (crcDest != crcOrig) goto _output_error; } } @@ -306,10 +309,10 @@ int basicTests(U32 seed, double compressibility) } DISPLAYLEVEL(3, "LZ4F_getFrameInfo on enough input : "); - iSize = 15 - iSize; + iSize = LZ4F_headerSize(ip, LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH); + CHECK( iSize ); CHECK( LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize) ); DISPLAYLEVEL(3, " correctly decoded \n"); - ip += iSize; } DISPLAYLEVEL(3, "Decode a buggy input : "); @@ -337,15 +340,16 @@ int basicTests(U32 seed, double compressibility) const BYTE* ip = (const BYTE*) compressedBuffer; const BYTE* const iend = ip + cSize; while (ip < iend) { - size_t oSize = oend-op; + size_t oSize = (size_t)(oend-op); size_t iSize = 1; CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); op += oSize; ip += iSize; } - { U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); - if (crcDest != crcOrig) goto _output_error; } - DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-ostart), COMPRESSIBLE_NOISE_LENGTH); + { U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); + if (crcDest != crcOrig) goto _output_error; + } + DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-ostart), (unsigned)COMPRESSIBLE_NOISE_LENGTH); } } @@ -379,8 +383,8 @@ int basicTests(U32 seed, double compressibility) while (ip < iend) { unsigned const nbBits = FUZ_rand(&randState) % maxBits; size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; - size_t oSize = oend-op; - if (iSize > (size_t)(iend-ip)) iSize = iend-ip; + size_t oSize = (size_t)(oend-op); + if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip); CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); op += oSize; ip += iSize; @@ -520,7 +524,7 @@ int basicTests(U32 seed, double compressibility) LZ4F_CDict* const cdict = LZ4F_createCDict(CNBuffer, dictSize); if (cdict == NULL) goto _output_error; CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) ); - + DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with NULL dict : "); CHECK_V(cSizeNoDict, LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity, @@ -657,6 +661,29 @@ int basicTests(U32 seed, double compressibility) CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL; } + DISPLAYLEVEL(3, "getBlockSize test: \n"); + { size_t result; + unsigned blockSizeID; + for (blockSizeID = 4; blockSizeID < 8; ++blockSizeID) { + result = LZ4F_getBlockSize(blockSizeID); + CHECK(result); + DISPLAYLEVEL(3, "Returned block size of %zu bytes for blockID %u \n", + result, blockSizeID); + } + + /* Test an invalid input that's too large */ + result = LZ4F_getBlockSize(8); + if(!LZ4F_isError(result) || + LZ4F_getErrorCode(result) != LZ4F_ERROR_maxBlockSize_invalid) + goto _output_error; + + /* Test an invalid input that's too small */ + result = LZ4F_getBlockSize(3); + if(!LZ4F_isError(result) || + LZ4F_getErrorCode(result) != LZ4F_ERROR_maxBlockSize_invalid) + goto _output_error; + } + DISPLAYLEVEL(3, "Skippable frame test : \n"); { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; @@ -755,17 +782,16 @@ static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, un int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration_s) { - unsigned testResult = 0; + int testResult = 0; unsigned testNb = 0; size_t const srcDataLength = 9 MB; /* needs to be > 2x4MB to test large blocks */ void* srcBuffer = NULL; - size_t const compressedBufferSize = LZ4F_compressFrameBound(srcDataLength, NULL); + size_t const compressedBufferSize = LZ4F_compressFrameBound(srcDataLength, NULL) + 4 MB; /* needs some margin */ void* compressedBuffer = NULL; void* decodedBuffer = NULL; U32 coreRand = seed; LZ4F_decompressionContext_t dCtx = NULL; LZ4F_compressionContext_t cCtx = NULL; - size_t result; clock_t const startClock = clock(); clock_t const clockDuration = duration_s * CLOCKS_PER_SEC; # undef CHECK @@ -773,10 +799,10 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } /* Create buffers */ - result = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); - CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result); - result = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION); - CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result); + { size_t const creationStatus = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); + CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); } + { size_t const creationStatus = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION); + CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); } srcBuffer = malloc(srcDataLength); CHECK(srcBuffer==NULL, "srcBuffer Allocation failed"); compressedBuffer = malloc(compressedBufferSize); @@ -829,37 +855,59 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi BYTE* op = (BYTE*)compressedBuffer; BYTE* const oend = op + (neverFlush ? LZ4F_compressFrameBound(srcSize, prefsPtr) : compressedBufferSize); /* when flushes are possible, can't guarantee a max compressed size */ unsigned const maxBits = FUZ_highbit((U32)srcSize); + size_t cSegmentSize; LZ4F_compressOptions_t cOptions; memset(&cOptions, 0, sizeof(cOptions)); - result = LZ4F_compressBegin(cCtx, op, oend-op, prefsPtr); - CHECK(LZ4F_isError(result), "Compression header failed (error %i)", (int)result); - op += result; + cSegmentSize = LZ4F_compressBegin(cCtx, op, (size_t)(oend-op), prefsPtr); + CHECK(LZ4F_isError(cSegmentSize), "Compression header failed (error %i)", + (int)cSegmentSize); + op += cSegmentSize; while (ip < iend) { unsigned const nbBitsSeg = FUZ_rand(&randState) % maxBits; size_t const sampleMax = (FUZ_rand(&randState) & ((1<<nbBitsSeg)-1)) + 1; size_t const iSize = MIN(sampleMax, (size_t)(iend-ip)); size_t const oSize = LZ4F_compressBound(iSize, prefsPtr); + size_t flushedSize; cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1); - DISPLAYLEVEL(6, "Sending %zi bytes to compress (stableSrc:%u) \n", - iSize, cOptions.stableSrc); + DISPLAYLEVEL(6, "Sending %u bytes to compress (stableSrc:%u) \n", + (unsigned)iSize, cOptions.stableSrc); - result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions); - CHECK(LZ4F_isError(result), "Compression failed (error %i : %s)", (int)result, LZ4F_getErrorName(result)); - op += result; + flushedSize = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions); + CHECK(LZ4F_isError(flushedSize), "Compression failed (error %i : %s)", + (int)flushedSize, LZ4F_getErrorName(flushedSize)); + op += flushedSize; ip += iSize; { unsigned const forceFlush = neverFlush ? 0 : ((FUZ_rand(&randState) & 3) == 1); if (forceFlush) { - result = LZ4F_flush(cCtx, op, oend-op, &cOptions); - CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result); - op += result; + size_t const flushSize = LZ4F_flush(cCtx, op, (size_t)(oend-op), &cOptions); + DISPLAYLEVEL(6,"flushing %u bytes \n", (unsigned)flushSize); + CHECK(LZ4F_isError(flushSize), "Compression failed (error %i)", (int)flushSize); + op += flushSize; } } } CHECK(op>=oend, "LZ4F_compressFrameBound overflow"); - result = LZ4F_compressEnd(cCtx, op, oend-op, &cOptions); - CHECK(LZ4F_isError(result), "Compression completion failed (error %i : %s)", (int)result, LZ4F_getErrorName(result)); - op += result; - cSize = op-(BYTE*)compressedBuffer; + { size_t const dstEndSafeSize = LZ4F_compressBound(0, prefsPtr); + int const tooSmallDstEnd = ((FUZ_rand(&randState) & 31) == 3); + size_t const dstEndTooSmallSize = (FUZ_rand(&randState) % dstEndSafeSize) + 1; + size_t const dstEndSize = tooSmallDstEnd ? dstEndTooSmallSize : dstEndSafeSize; + BYTE const canaryByte = (BYTE)(FUZ_rand(&randState) & 255); + size_t flushedSize; + DISPLAYLEVEL(7,"canaryByte at pos %u / %u \n", + (unsigned)((size_t)(op - (BYTE*)compressedBuffer) + dstEndSize), + (unsigned)compressedBufferSize); + assert(op + dstEndSize < (BYTE*)compressedBuffer + compressedBufferSize); + op[dstEndSize] = canaryByte; + flushedSize = LZ4F_compressEnd(cCtx, op, dstEndSize, &cOptions); + CHECK(op[dstEndSize] != canaryByte, "LZ4F_compressEnd writes beyond dstCapacity !"); + if (LZ4F_isError(flushedSize)) { + if (tooSmallDstEnd) /* failure is allowed */ continue; + CHECK(1, "Compression completion failed (error %i : %s)", + (int)flushedSize, LZ4F_getErrorName(flushedSize)); + } + op += flushedSize; + } + cSize = (size_t)(op - (BYTE*)compressedBuffer); DISPLAYLEVEL(5, "\nCompressed %u bytes into %u \n", (U32)srcSize, (U32)cSize); } @@ -872,8 +920,10 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi unsigned const maxBits = MAX(3, suggestedBits); unsigned const nonContiguousDst = FUZ_rand(&randState) % 3; /* 0 : contiguous; 1 : non-contiguous; 2 : dst overwritten */ size_t totalOut = 0; + size_t decSize = 0; XXH64_state_t xxh64; XXH64_reset(&xxh64, 1); + assert(ip < iend); while (ip < iend) { unsigned const nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1; unsigned const nbBitsO = (FUZ_rand(&randState) % (maxBits)) + 1; @@ -885,10 +935,11 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi memset(&dOptions, 0, sizeof(dOptions)); dOptions.stableDst = FUZ_rand(&randState) & 1; if (nonContiguousDst==2) dOptions.stableDst = 0; /* overwrite mode */ - result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions); - if (LZ4F_getErrorCode(result) == LZ4F_ERROR_contentChecksum_invalid) + decSize = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions); + if (LZ4F_getErrorCode(decSize) == LZ4F_ERROR_contentChecksum_invalid) locateBuffDiff(srcStart, decodedBuffer, srcSize, nonContiguousDst); - CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s)", (int)result, LZ4F_getErrorName(result)); + CHECK(LZ4F_isError(decSize), "Decompression failed (error %i:%s)", + (int)decSize, LZ4F_getErrorName(decSize)); XXH64_update(&xxh64, op, (U32)oSize); totalOut += oSize; op += oSize; @@ -896,7 +947,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi op += nonContiguousDst; if (nonContiguousDst==2) op = (BYTE*)decodedBuffer; /* overwritten destination */ } - CHECK(result != 0, "Frame decompression failed (error %i)", (int)result); + CHECK(decSize != 0, "Frame decompression failed (error %i)", (int)decSize); if (totalOut) { /* otherwise, it's a skippable frame */ U64 const crcDecoded = XXH64_digest(&xxh64); if (crcDecoded != crcOrig) locateBuffDiff(srcStart, decodedBuffer, srcSize, nonContiguousDst); diff --git a/tests/fullbench.c b/tests/fullbench.c index fd1202d..1a52aab 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -42,6 +42,7 @@ #include <string.h> /* strcmp */ #include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */ +#define LZ4_DISABLE_DEPRECATE_WARNINGS /* LZ4_decompress_fast */ #include "lz4.h" #include "lz4hc.h" #include "lz4frame.h" @@ -160,12 +161,14 @@ static size_t BMK_findMaxMem(U64 requiredMem) static LZ4_stream_t LZ4_stream; static void local_LZ4_resetDictT(void) { - LZ4_resetStream(&LZ4_stream); + void* const r = LZ4_initStream(&LZ4_stream, sizeof(LZ4_stream)); + assert(r != NULL); } static void local_LZ4_createStream(void) { - LZ4_resetStream(&LZ4_stream); + void* const r = LZ4_initStream(&LZ4_stream, sizeof(LZ4_stream)); + assert(r != NULL); } static int local_LZ4_saveDict(const char* in, char* out, int inSize) @@ -242,7 +245,7 @@ static int local_LZ4_compress_forceDict(const char* in, char* out, int inSize) LZ4_streamHC_t LZ4_streamHC; static void local_LZ4_resetStreamHC(void) { - LZ4_resetStreamHC(&LZ4_streamHC, 0); + LZ4_initStreamHC(&LZ4_streamHC, sizeof(LZ4_streamHC)); } static int local_LZ4_saveDictHC(const char* in, char* out, int inSize) @@ -326,16 +329,19 @@ static int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSi /* frame functions */ static int local_LZ4F_compressFrame(const char* in, char* out, int inSize) { - return (int)LZ4F_compressFrame(out, LZ4F_compressFrameBound(inSize, NULL), in, inSize, NULL); + assert(inSize >= 0); + return (int)LZ4F_compressFrame(out, LZ4F_compressFrameBound((size_t)inSize, NULL), in, (size_t)inSize, NULL); } static LZ4F_decompressionContext_t g_dCtx; static int local_LZ4F_decompress(const char* in, char* out, int inSize, int outSize) { - size_t srcSize = inSize; - size_t dstSize = outSize; + size_t srcSize = (size_t)inSize; + size_t dstSize = (size_t)outSize; size_t result; + assert(inSize >= 0); + assert(outSize >= 0); result = LZ4F_decompress(g_dCtx, out, &dstSize, in, &srcSize, NULL); if (result!=0) { DISPLAY("Error decompressing frame : unfinished frame\n"); exit(8); } if (srcSize != (size_t)inSize) { DISPLAY("Error decompressing frame : read size incorrect\n"); exit(9); } @@ -367,7 +373,6 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) size_t readSize; int compressedBuffSize; U32 crcOriginal; - size_t errorCode; /* Check file existence */ if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } @@ -384,7 +389,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) /* Allocation */ chunkP = (struct chunkParameters*) malloc(((benchedSize / (size_t)g_chunkSize)+1) * sizeof(struct chunkParameters)); orig_buff = (char*) malloc(benchedSize); - nbChunks = (int) ((benchedSize + (g_chunkSize-1)) / g_chunkSize); + nbChunks = (int) ((benchedSize + (size_t)g_chunkSize - 1) / (size_t)g_chunkSize); maxCompressedChunkSize = LZ4_compressBound(g_chunkSize); compressedBuffSize = nbChunks * maxCompressedChunkSize; compressed_buff = (char*)malloc((size_t)compressedBuffSize); @@ -439,7 +444,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) char* out = compressed_buff; nbChunks = (int) (((int)benchedSize + (g_chunkSize-1))/ g_chunkSize); for (i=0; i<nbChunks; i++) { - chunkP[i].id = i; + chunkP[i].id = (U32)i; chunkP[i].origBuffer = in; in += g_chunkSize; if ((int)remaining > g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; @@ -561,7 +566,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) case 8: decompressionFunction = local_LZ4_decompress_safe_forceExtDict; dName = "LZ4_decompress_safe_forceExtDict"; break; #endif case 9: decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress"; - errorCode = LZ4F_compressFrame(compressed_buff, compressedBuffSize, orig_buff, benchedSize, NULL); + { size_t const errorCode = LZ4F_compressFrame(compressed_buff, compressedBuffSize, orig_buff, benchedSize, NULL); if (LZ4F_isError(errorCode)) { DISPLAY("Error while preparing compressed frame\n"); free(orig_buff); @@ -573,6 +578,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) chunkP[0].compressedSize = (int)errorCode; nbChunks = 1; break; + } default : continue; /* skip if unknown ID */ } @@ -610,7 +616,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) PROGRESS("%2i-%-34.34s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000000); /* CRC Checking */ - crcDecoded = XXH32(orig_buff, (int)benchedSize, 0); + crcDecoded = XXH32(orig_buff, benchedSize, 0); if (checkResult && (crcOriginal!=crcDecoded)) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOriginal, (unsigned)crcDecoded); diff --git a/tests/fuzzer.c b/tests/fuzzer.c index b29e82e..3128e6d 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -32,13 +32,13 @@ # pragma warning(disable : 4310) /* disable: C4310: constant char value > 127 */ #endif -#define LZ4_DISABLE_DEPRECATE_WARNINGS - /*-************************************ * Dependencies **************************************/ #if defined(__unix__) && !defined(_AIX) /* must be included before platform.h for MAP_ANONYMOUS */ +# undef _GNU_SOURCE /* in case it's already defined */ +# define _GNU_SOURCE /* MAP_ANONYMOUS even in -std=c99 mode */ # include <sys/mman.h> /* mmap */ #endif #include "platform.h" /* _CRT_SECURE_NO_WARNINGS */ @@ -48,11 +48,11 @@ #include <string.h> /* strcmp */ #include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */ #include <assert.h> -#if defined(__unix__) && defined(_AIX) -# include <sys/mman.h> /* mmap */ -#endif +#include <limits.h> /* INT_MAX */ +#define LZ4_DISABLE_DEPRECATE_WARNINGS /* LZ4_decompress_fast */ #define LZ4_STATIC_LINKING_ONLY +#include "lz4.h" #define LZ4_HC_STATIC_LINKING_ONLY #include "lz4hc.h" #define XXH_STATIC_LINKING_ONLY @@ -145,10 +145,10 @@ static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, dou /* Select : Literal (noise) or copy (within 64K) */ if (FUZ_RAND15BITS < P32) { /* Copy (within 64K) */ - size_t const length = FUZ_RANDLENGTH + 4; + size_t const length = (size_t)FUZ_RANDLENGTH + 4; size_t const d = MIN(pos+length, bufferSize); size_t match; - size_t offset = FUZ_RAND15BITS + 1; + size_t offset = (size_t)FUZ_RAND15BITS + 1; while (offset > pos) offset >>= 1; match = pos - offset; while (pos < d) BBuffer[pos++] = BBuffer[match++]; @@ -308,13 +308,13 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c unsigned long long hcbytes = 0; unsigned long long ccbytes = 0; void* const CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); - size_t const compressedBufferSize = LZ4_compressBound(FUZ_MAX_BLOCK_SIZE); + size_t const compressedBufferSize = (size_t)LZ4_compressBound(FUZ_MAX_BLOCK_SIZE); char* const compressedBuffer = (char*)malloc(compressedBufferSize); char* const decodedBuffer = (char*)malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE); size_t const labSize = 96 KB; void* const lowAddrBuffer = FUZ_createLowAddr(labSize); - void* const stateLZ4 = malloc(LZ4_sizeofState()); - void* const stateLZ4HC = malloc(LZ4_sizeofStateHC()); + void* const stateLZ4 = malloc((size_t)LZ4_sizeofState()); + void* const stateLZ4HC = malloc((size_t)LZ4_sizeofStateHC()); LZ4_stream_t LZ4dict; LZ4_streamHC_t LZ4dictHC; U32 coreRandState = seed; @@ -345,7 +345,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c DISPLAY("Not enough memory to start fuzzer tests"); goto _output_error; } - memset(&LZ4dict, 0, sizeof(LZ4dict)); + if ( LZ4_initStream(&LZ4dict, sizeof(LZ4dict)) == NULL) abort(); + if ( LZ4_initStreamHC(&LZ4dictHC, sizeof(LZ4dictHC)) == NULL) abort(); { U32 randState = coreRandState ^ PRIME3; FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); } @@ -369,7 +370,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c const char* dict = block - dictSize; int compressedSize, HCcompressedSize; int blockContinueCompressedSize; - U32 const crcOrig = XXH32(block, blockSize, 0); + U32 const crcOrig = XXH32(block, (size_t)blockSize, 0); int ret; FUZ_displayUpdate(cycleNb); @@ -394,7 +395,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c DISPLAYLEVEL(5, "destSize : %7i/%7i; content%7i/%7i ", ret, targetSize, srcSize, blockSize); if (targetSize>0) { /* check correctness */ - U32 const crcBase = XXH32(block, srcSize, 0); + U32 const crcBase = XXH32(block, (size_t)srcSize, 0); char const canary = FUZ_rand(&randState) & 255; FUZ_CHECKTEST((ret==0), "LZ4_compress_destSize() compression failed"); FUZ_DISPLAYTEST(); @@ -429,7 +430,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_HC_destSize() fed more than src buffer !"); if (targetSize>0) { /* check correctness */ - U32 const crcBase = XXH32(block, srcSize, 0); + U32 const crcBase = XXH32(block, (size_t)srcSize, 0); char const canary = FUZ_rand(&randState) & 255; FUZ_CHECKTEST((ret==0), "LZ4_compress_HC_destSize() compression failed"); FUZ_DISPLAYTEST(); @@ -508,7 +509,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Test decoding with a one byte input */ FUZ_DISPLAYTEST("LZ4_decompress_safe() with one byte input"); - { char const tmp = 0xFF; + { char const tmp = (char)0xFF; LZ4_decompress_safe(&tmp, decodedBuffer, 1, blockSize); } @@ -518,7 +519,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* 14 bytes of literals, followed by a 14 byte match. * Should not read beyond the end of the buffer. * See https://github.com/lz4/lz4/issues/508. */ - *tmp = 0xEE; + *tmp = (char)0xEE; memset(tmp + 1, 0, 14); tmp[15] = 14; tmp[16] = 0; @@ -546,7 +547,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite amply sufficient space"); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data"); FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ4_decompress_safe overrun specified output buffer size"); - { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); } @@ -581,7 +582,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Test partial decoding => must work */ FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial"); { size_t const missingBytes = FUZ_rand(&randState) % blockSize; - int const targetSize = (int)(blockSize - missingBytes); + int const targetSize = (int)((size_t)blockSize - missingBytes); char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A; int const decResult = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, targetSize, blockSize); FUZ_CHECKTEST(decResult<0, "LZ4_decompress_safe_partial failed despite valid input data (error:%i)", decResult); @@ -641,7 +642,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Compress using dictionary */ FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary of size %i", dictSize); { LZ4_stream_t LZ4_stream; - LZ4_resetStream(&LZ4_stream); + LZ4_initStream(&LZ4_stream, sizeof(LZ4_stream)); LZ4_compress_fast_continue (&LZ4_stream, dict, compressedBuffer, dictSize, (int)compressedBufferSize, 1); /* Just to fill hash tables */ blockContinueCompressedSize = LZ4_compress_fast_continue (&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue failed"); @@ -652,7 +653,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c memcpy(decodedBuffer, dict, dictSize); ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer+dictSize, blockSize, decodedBuffer, dictSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input"); - { U32 const crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); + { U32 const crcCheck = XXH32(decodedBuffer+dictSize, (size_t)blockSize, 0); if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); } @@ -660,13 +661,13 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_DISPLAYTEST("test LZ4_decompress_safe_usingDict()"); ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); - { U32 const crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); + { U32 const crcCheck = XXH32(decodedBuffer+dictSize, (size_t)blockSize, 0); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); } /* Compress using External dictionary */ FUZ_DISPLAYTEST("test LZ4_compress_fast_continue(), with non-contiguous dictionary"); - dict -= (FUZ_rand(&randState) & 0xF) + 1; /* create space, so now dictionary is an ExtDict */ + dict -= (size_t)(FUZ_rand(&randState) & 0xF) + 1; /* create space, so now dictionary is an ExtDict */ if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; LZ4_loadDict(&LZ4dict, dict, dictSize); blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1); @@ -678,7 +679,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize); FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary loaded with LZ4_loadDict()"); - DISPLAYLEVEL(5, " compress %i bytes from buffer(%p) into dst(%p) using dict(%p) of size %i \n", blockSize, block, decodedBuffer, dict, dictSize); + DISPLAYLEVEL(5, " compress %i bytes from buffer(%p) into dst(%p) using dict(%p) of size %i \n", + blockSize, (const void *)block, (void *)decodedBuffer, (const void *)dict, dictSize); LZ4_loadDict(&LZ4dict, dict, dictSize); ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); @@ -686,7 +688,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Decompress with dictionary as external */ FUZ_DISPLAYTEST("test LZ4_decompress_fast_usingDict() with dictionary as extDict"); - DISPLAYLEVEL(5, " decoding %i bytes from buffer(%p) using dict(%p) of size %i \n", blockSize, decodedBuffer, dict, dictSize); + DISPLAYLEVEL(5, " decoding %i bytes from buffer(%p) using dict(%p) of size %i \n", + blockSize, (void *)decodedBuffer, (const void *)dict, dictSize); decodedBuffer[blockSize] = 0; ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input"); @@ -740,10 +743,11 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary()"); LZ4_loadDict(&LZ4dict, dict, dictSize); - LZ4_resetStream(&LZ4_stream); + LZ4_initStream(&LZ4_stream, sizeof(LZ4_stream)); LZ4_attach_dictionary(&LZ4_stream, &LZ4dict); blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue using extDictCtx failed"); + FUZ_CHECKTEST(LZ4_stream.internal_donotuse.dirty, "context should be good"); /* In the future, it might be desirable to let extDictCtx mode's * output diverge from the output generated by regular extDict mode. @@ -754,19 +758,21 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(XXH32(compressedBuffer, blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output"); FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary(), but output buffer is 1 byte too short"); - LZ4_resetStream(&LZ4_stream); + LZ4_resetStream_fast(&LZ4_stream); LZ4_attach_dictionary(&LZ4_stream, &LZ4dict); ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1); FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using extDictCtx should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize); + /* note : context is no longer dirty after a failed compressed block */ FUZ_DISPLAYTEST(); - LZ4_resetStream(&LZ4_stream); + LZ4_resetStream_fast(&LZ4_stream); LZ4_attach_dictionary(&LZ4_stream, &LZ4dict); ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx should work : enough size available within output buffer"); FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output"); FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output"); + FUZ_CHECKTEST(LZ4_stream.internal_donotuse.dirty, "context should be good"); FUZ_DISPLAYTEST(); LZ4_resetStream_fast(&LZ4_stream); @@ -776,6 +782,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx with re-used context should work : enough size available within output buffer"); FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output"); FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output"); + FUZ_CHECKTEST(LZ4_stream.internal_donotuse.dirty, "context should be good"); } /* Decompress with dictionary as external */ @@ -794,7 +801,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); - { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); } @@ -823,29 +830,31 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_DISPLAYTEST("LZ4_compress_HC_continue with an external dictionary"); dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */ if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; - LZ4_resetStreamHC (&LZ4dictHC, compressionLevel); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); - LZ4_setCompressionLevel(&LZ4dictHC, compressionLevel-1); + LZ4_setCompressionLevel (&LZ4dictHC, compressionLevel); blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue failed"); + FUZ_CHECKTEST(LZ4dictHC.internal_donotuse.dirty, "Context should be clean"); - FUZ_DISPLAYTEST(); + FUZ_DISPLAYTEST("LZ4_compress_HC_continue with same external dictionary, but output buffer 1 byte too short"); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); - FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDict should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize); + FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDict should fail : one missing byte for output buffer (expected %i, but result=%i)", blockContinueCompressedSize, ret); + /* note : context is no longer dirty after a failed compressed block */ - FUZ_DISPLAYTEST(); + FUZ_DISPLAYTEST("LZ4_compress_HC_continue with same external dictionary, and output buffer exactly the right size"); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize); - FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue size is different (%i != %i)", ret, blockContinueCompressedSize); + FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue size is different : ret(%i) != expected(%i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue should work : enough size available within output buffer"); + FUZ_CHECKTEST(LZ4dictHC.internal_donotuse.dirty, "Context should be clean"); FUZ_DISPLAYTEST(); decodedBuffer[blockSize] = 0; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); - { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0); if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); } @@ -854,26 +863,29 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_DISPLAYTEST(); { LZ4_streamHC_t LZ4_streamHC; + LZ4_initStreamHC(&LZ4_streamHC, sizeof(LZ4_streamHC)); - LZ4_resetStreamHC (&LZ4dictHC, compressionLevel); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); - LZ4_resetStreamHC (&LZ4_streamHC, compressionLevel); LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC); + LZ4_setCompressionLevel (&LZ4_streamHC, compressionLevel); blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, (int)compressedBufferSize); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue with ExtDictCtx failed"); + FUZ_CHECKTEST(LZ4_streamHC.internal_donotuse.dirty, "Context should be clean"); FUZ_DISPLAYTEST(); - LZ4_resetStreamHC (&LZ4_streamHC, compressionLevel); + LZ4_resetStreamHC_fast (&LZ4_streamHC, compressionLevel); LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC); ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDictCtx should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize); + /* note : context is no longer dirty after a failed compressed block */ FUZ_DISPLAYTEST(); - LZ4_resetStreamHC (&LZ4_streamHC, compressionLevel); + LZ4_resetStreamHC_fast (&LZ4_streamHC, compressionLevel); LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC); ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue using ExtDictCtx should work : enough size available within output buffer"); + FUZ_CHECKTEST(LZ4_streamHC.internal_donotuse.dirty, "Context should be clean"); FUZ_DISPLAYTEST(); LZ4_resetStreamHC_fast (&LZ4_streamHC, compressionLevel); @@ -881,25 +893,26 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx and fast reset size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue using ExtDictCtx and fast reset should work : enough size available within output buffer"); + FUZ_CHECKTEST(LZ4_streamHC.internal_donotuse.dirty, "Context should be clean"); + } - FUZ_DISPLAYTEST(); - decodedBuffer[blockSize] = 0; - ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); - FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); - FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); - { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); - if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); - } + FUZ_DISPLAYTEST(); + decodedBuffer[blockSize] = 0; + ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); + FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); + FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); + { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0); + if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); } /* Compress HC continue destSize */ FUZ_DISPLAYTEST(); - { int const availableSpace = (FUZ_rand(&randState) % blockSize) + 5; + { int const availableSpace = (int)(FUZ_rand(&randState) % blockSize) + 5; int consumedSize = blockSize; FUZ_DISPLAYTEST(); - LZ4_resetStreamHC (&LZ4dictHC, compressionLevel); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); + LZ4_setCompressionLevel(&LZ4dictHC, compressionLevel); blockContinueCompressedSize = LZ4_compress_HC_continue_destSize(&LZ4dictHC, block, compressedBuffer, &consumedSize, availableSpace); DISPLAYLEVEL(5, " LZ4_compress_HC_continue_destSize : compressed %6i/%6i into %6i/%6i at cLevel=%i\n", consumedSize, blockSize, blockContinueCompressedSize, availableSpace, compressionLevel); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue_destSize failed"); @@ -911,8 +924,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, consumedSize, dict, dictSize); FUZ_CHECKTEST(ret!=consumedSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); FUZ_CHECKTEST(decodedBuffer[consumedSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size") - { U32 const crcSrc = XXH32(block, consumedSize, 0); - U32 const crcDst = XXH32(decodedBuffer, consumedSize, 0); + { U32 const crcSrc = XXH32(block, (size_t)consumedSize, 0); + U32 const crcDst = XXH32(decodedBuffer, (size_t)consumedSize, 0); if (crcSrc!=crcDst) FUZ_findDiff(block, decodedBuffer); FUZ_CHECKTEST(crcSrc!=crcDst, "LZ4_decompress_safe_usingDict corrupted decoded data"); } @@ -987,9 +1000,10 @@ static void FUZ_unitTests(int compressionLevel) /* simple compression test */ crcOrig = XXH64(testInput, testCompressedSize, 0); - LZ4_resetStream(&streamingState); + LZ4_initStream(&streamingState, sizeof(streamingState)); result = LZ4_compress_fast_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1, 1); FUZ_CHECKTEST(result==0, "LZ4_compress_fast_continue() compression failed!"); + FUZ_CHECKTEST(streamingState.internal_donotuse.dirty, "context should be clean") result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize); FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); @@ -1012,7 +1026,7 @@ static void FUZ_unitTests(int compressionLevel) XXH64_reset(&xxhOrig, 0); XXH64_reset(&xxhNewSafe, 0); XXH64_reset(&xxhNewFast, 0); - LZ4_resetStream(&streamingState); + LZ4_resetStream_fast(&streamingState); LZ4_setStreamDecode(&decodeStateSafe, NULL, 0); LZ4_setStreamDecode(&decodeStateFast, NULL, 0); @@ -1050,69 +1064,112 @@ static void FUZ_unitTests(int compressionLevel) } /* LZ4 HC streaming tests */ - { LZ4_streamHC_t* sp; - LZ4_streamHC_t sHC; + { LZ4_streamHC_t sHC; /* statically allocated */ U64 crcOrig; int result; + LZ4_initStreamHC(&sHC, sizeof(sHC)); /* Allocation test */ - sp = LZ4_createStreamHC(); - FUZ_CHECKTEST(sp==NULL, "LZ4_createStreamHC() allocation failed"); - LZ4_freeStreamHC(sp); + DISPLAYLEVEL(3, " Basic HC allocation : "); + { LZ4_streamHC_t* const sp = LZ4_createStreamHC(); + FUZ_CHECKTEST(sp==NULL, "LZ4_createStreamHC() allocation failed"); + LZ4_freeStreamHC(sp); + } + DISPLAYLEVEL(3, " OK \n"); /* simple HC compression test */ - crcOrig = XXH64(testInput, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, compressionLevel); - result = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); - - result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); - { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); } + DISPLAYLEVEL(3, " Simple HC round-trip : "); + { U64 const crc64 = XXH64(testInput, testCompressedSize, 0); + LZ4_setCompressionLevel(&sHC, compressionLevel); + result = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1); + FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); + + result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize); + FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); + { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() decompression corruption"); + } } + DISPLAYLEVEL(3, " OK \n"); + + /* long sequence test */ + DISPLAYLEVEL(3, " Long sequence HC test : "); + { size_t const blockSize = 1 MB; + size_t const targetSize = 4116; /* size carefully selected to trigger an overflow */ + void* const block = malloc(blockSize); + void* const dstBlock = malloc(targetSize+1); + BYTE const sentinel = 101; + int srcSize; + + assert(block != NULL); assert(dstBlock != NULL); + memset(block, 0, blockSize); + ((char*)dstBlock)[targetSize] = sentinel; + + LZ4_resetStreamHC_fast(&sHC, 3); + assert(blockSize < INT_MAX); + srcSize = (int)blockSize; + assert(targetSize < INT_MAX); + result = LZ4_compress_HC_destSize(&sHC, (const char*)block, (char*)dstBlock, &srcSize, (int)targetSize, 3); + DISPLAYLEVEL(4, "cSize=%i; readSize=%i; ", result, srcSize); + FUZ_CHECKTEST(result!=4116, "LZ4_compress_HC_destSize() : compression must fill dstBuffer completely, but no more !"); + FUZ_CHECKTEST(((char*)dstBlock)[targetSize] != sentinel, "LZ4_compress_HC_destSize()") + + LZ4_resetStreamHC_fast(&sHC, 3); /* make sure the context is clean after the test */ + free(block); + free(dstBlock); + } + DISPLAYLEVEL(3, " OK \n"); /* simple dictionary HC compression test */ - crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, compressionLevel); - LZ4_loadDictHC(&sHC, testInput, 64 KB); - result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); - - result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 64 KB); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed"); - { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() simple dictionary decompression test : corruption"); } + DISPLAYLEVEL(3, " HC dictionary compression test : "); + { U64 const crc64 = XXH64(testInput + 64 KB, testCompressedSize, 0); + LZ4_resetStreamHC_fast(&sHC, compressionLevel); + LZ4_loadDictHC(&sHC, testInput, 64 KB); + result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); + FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); + + result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 64 KB); + FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed"); + { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() simple dictionary decompression test : corruption"); + } } + DISPLAYLEVEL(3, " OK \n"); /* multiple HC compression test with dictionary */ { int result1, result2; int segSize = testCompressedSize / 2; - crcOrig = XXH64(testInput + segSize, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, compressionLevel); + U64 const crc64 = XXH64(testInput + segSize, testCompressedSize, 0); + LZ4_resetStreamHC_fast(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, segSize); result1 = LZ4_compress_HC_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1); FUZ_CHECKTEST(result1==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result1); - result2 = LZ4_compress_HC_continue(&sHC, testInput + 2*segSize, testCompressed+result1, segSize, segSize-1); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); + result2 = LZ4_compress_HC_continue(&sHC, testInput + 2*(size_t)segSize, testCompressed+result1, segSize, segSize-1); FUZ_CHECKTEST(result2==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result2); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result1, segSize, testInput, segSize); FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 1 failed"); result = LZ4_decompress_safe_usingDict(testCompressed+result1, testVerify+segSize, result2, segSize, testInput, 2*segSize); FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 2 failed"); - { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); } - } + { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); + } } /* remote dictionary HC compression test */ - crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, compressionLevel); - LZ4_loadDictHC(&sHC, testInput, 32 KB); - result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result); - - result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 32 KB); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe_usingDict() decompression failed following remote dictionary HC compression test"); - { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() decompression corruption"); } + { U64 const crc64 = XXH64(testInput + 64 KB, testCompressedSize, 0); + LZ4_resetStreamHC_fast(&sHC, compressionLevel); + LZ4_loadDictHC(&sHC, testInput, 32 KB); + result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); + FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); + + result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 32 KB); + FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe_usingDict() decompression failed following remote dictionary HC compression test"); + { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe_usingDict() decompression corruption"); + } } /* multiple HC compression with ext. dictionary */ { XXH64_state_t crcOrigState; @@ -1121,11 +1178,11 @@ static void FUZ_unitTests(int compressionLevel) int dictSize = (FUZ_rand(&randState) & 8191); char* dst = testVerify; - size_t segStart = dictSize + 7; + size_t segStart = (size_t)dictSize + 7; int segSize = (FUZ_rand(&randState) & 8191); int segNb = 1; - LZ4_resetStreamHC(&sHC, compressionLevel); + LZ4_resetStreamHC_fast(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, dict, dictSize); XXH64_reset(&crcOrigState, 0); @@ -1136,6 +1193,7 @@ static void FUZ_unitTests(int compressionLevel) crcOrig = XXH64_digest(&crcOrigState); result = LZ4_compress_HC_continue(&sHC, testInput + segStart, testCompressed, segSize, LZ4_compressBound(segSize)); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); result = LZ4_decompress_safe_usingDict(testCompressed, dst, result, segSize, dict, dictSize); FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %i failed", segNb); @@ -1145,13 +1203,14 @@ static void FUZ_unitTests(int compressionLevel) FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() part %i corruption", segNb); } + assert(segSize >= 0); dict = dst; dictSize = segSize; - dst += segSize + 1; + dst += (size_t)segSize + 1; segNb ++; - segStart += segSize + (FUZ_rand(&randState) & 0xF) + 1; + segStart += (size_t)segSize + (FUZ_rand(&randState) & 0xF) + 1; segSize = (FUZ_rand(&randState) & 8191); } } @@ -1172,7 +1231,7 @@ static void FUZ_unitTests(int compressionLevel) XXH64_reset(&xxhOrig, 0); XXH64_reset(&xxhNewSafe, 0); XXH64_reset(&xxhNewFast, 0); - LZ4_resetStreamHC(&sHC, compressionLevel); + LZ4_resetStreamHC_fast(&sHC, compressionLevel); LZ4_setStreamDecode(&decodeStateSafe, NULL, 0); LZ4_setStreamDecode(&decodeStateFast, NULL, 0); @@ -1183,6 +1242,7 @@ static void FUZ_unitTests(int compressionLevel) memcpy (ringBuffer + rNext, testInput + iNext, messageSize); compressedSize = LZ4_compress_HC_continue(&sHC, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed"); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, messageSize); FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe_continue() test failed"); @@ -1229,11 +1289,11 @@ static void FUZ_unitTests(int compressionLevel) int dNext = 0; int compressedSize; - assert((size_t)(dBufferSize + 1 + dBufferSize) < testVerifySize); /* space used by ringBufferSafe and ringBufferFast */ + assert((size_t)dBufferSize * 2 + 1 < testVerifySize); /* space used by ringBufferSafe and ringBufferFast */ XXH64_reset(&xxhOrig, 0); XXH64_reset(&xxhNewSafe, 0); XXH64_reset(&xxhNewFast, 0); - LZ4_resetStreamHC(&sHC, compressionLevel); + LZ4_resetStreamHC_fast(&sHC, compressionLevel); LZ4_setStreamDecode(&decodeStateSafe, NULL, 0); LZ4_setStreamDecode(&decodeStateFast, NULL, 0); @@ -1246,6 +1306,7 @@ static void FUZ_unitTests(int compressionLevel) compressedSize = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed"); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, ringBufferSafe + dNext, compressedSize, messageSize); FUZ_CHECKTEST(result!=messageSize, "64K D.ringBuffer : LZ4_decompress_safe_continue() test failed"); @@ -1263,7 +1324,8 @@ static void FUZ_unitTests(int compressionLevel) /* prepare second message */ dNext += messageSize; - totalMessageSize += messageSize; + assert(messageSize >= 0); + totalMessageSize += (unsigned)messageSize; messageSize = maxMessageSize; iNext = BSIZE1+1; assert(BSIZE1 >= 65535); @@ -1277,6 +1339,7 @@ static void FUZ_unitTests(int compressionLevel) compressedSize = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed"); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); DISPLAYLEVEL(5, "compressed %i bytes to %i bytes \n", messageSize, compressedSize); /* test LZ4_decompress_safe_continue */ diff --git a/tests/test_custom_block_sizes.sh b/tests/test_custom_block_sizes.sh new file mode 100755 index 0000000..aba6733 --- /dev/null +++ b/tests/test_custom_block_sizes.sh @@ -0,0 +1,72 @@ +#/usr/bin/env sh +set -e + +LZ4=../lz4 +CHECKFRAME=./checkFrame +DATAGEN=./datagen + +failures="" + +TMPFILE=/tmp/test_custom_block_sizes.$$ +TMPFILE1=/tmp/test_custom_block_sizes1.$$ +TMPFILE2=/tmp/test_custom_block_sizes2.$$ +$DATAGEN -g12345678 > $TMPFILE1 +$DATAGEN -g12345678 > $TMPFILE2 + +echo Testing -B31 +$LZ4 -f -B31 $TMPFILE1 && failures="31 (should fail) " + +for blocksize in 32 65535 65536 +do + echo Testing -B$blocksize + $LZ4 -f -B$blocksize $TMPFILE1 + $LZ4 -f -B$blocksize $TMPFILE2 + cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4 + $CHECKFRAME -B$blocksize -b4 $TMPFILE.lz4 || failures="$failures $blocksize " +done + +for blocksize in 65537 262143 262144 +do + echo Testing -B$blocksize + $LZ4 -f -B$blocksize $TMPFILE1 + $LZ4 -f -B$blocksize $TMPFILE2 + cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4 + $CHECKFRAME -B$blocksize -b5 $TMPFILE.lz4 || failures="$failures $blocksize " +done + +for blocksize in 262145 1048575 1048576 +do + echo Testing -B$blocksize + $LZ4 -f -B$blocksize $TMPFILE1 + $LZ4 -f -B$blocksize $TMPFILE2 + cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4 + $CHECKFRAME -B$blocksize -b6 $TMPFILE.lz4 || failures="$failures $blocksize " +done + +for blocksize in 1048577 4194303 4194304 +do + echo Testing -B$blocksize + $LZ4 -f -B$blocksize $TMPFILE1 + $LZ4 -f -B$blocksize $TMPFILE2 + cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4 + $CHECKFRAME -B$blocksize -b7 $TMPFILE.lz4 || failures="$failures $blocksize " +done + +for blocksize in 4194305 10485760 +do + echo Testing -B$blocksize + $LZ4 -f -B$blocksize $TMPFILE1 + $LZ4 -f -B$blocksize $TMPFILE2 + cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4 + $CHECKFRAME -B4194304 -b7 $TMPFILE.lz4 || failures="$failures $blocksize " +done + +rm $TMPFILE.lz4 $TMPFILE1 $TMPFILE1.lz4 $TMPFILE2 $TMPFILE2.lz4 +if [ "$failures" == "" ] +then + echo ---- All tests passed + exit 0 +else + echo ---- The following tests had failures: $failures + exit 1 +fi |