diff options
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/Makefile | 16 | ||||
| -rw-r--r-- | tests/checkFrame.c | 306 | ||||
| -rw-r--r-- | tests/frametest.c | 27 | ||||
| -rw-r--r-- | tests/fullbench.c | 4 | ||||
| -rw-r--r-- | tests/fuzzer.c | 43 | ||||
| -rwxr-xr-x | tests/test_custom_block_sizes.sh | 72 |
6 files changed, 444 insertions, 24 deletions
diff --git a/tests/Makefile b/tests/Makefile index 3de111b..24dc581 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -63,7 +63,7 @@ NB_LOOPS ?= -i1 default: all -all: fullbench fuzzer frametest roundTripTest datagen +all: fullbench fuzzer frametest roundTripTest datagen checkFrame all32: CFLAGS+=-m32 all32: all @@ -109,6 +109,9 @@ roundTripTest : lz4.o lz4hc.o xxhash.o roundTripTest.c datagen : $(PRGDIR)/datagen.c datagencli.c $(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT) +checkFrame : lz4frame.o lz4.o lz4hc.o xxhash.o checkFrame.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + clean: @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) @$(MAKE) -C $(PRGDIR) $@ > $(VOID) @@ -118,7 +121,8 @@ clean: fuzzer$(EXT) fuzzer32$(EXT) \ frametest$(EXT) frametest32$(EXT) \ fasttest$(EXT) roundTripTest$(EXT) \ - datagen$(EXT) checkTag$(EXT) + datagen$(EXT) checkTag$(EXT) \ + frameTest$(EXT) @rm -fR $(TESTDIR) @echo Cleaning completed @@ -148,11 +152,17 @@ endif DD:=dd -test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-install +test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-install test-amalgamation test32: CFLAGS+=-m32 test32: test +test-amalgamation: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c + cat $(LZ4DIR)/lz4.c > lz4_all.c + cat $(LZ4DIR)/lz4hc.c >> lz4_all.c + $(CC) -I$(LZ4DIR) -c lz4_all.c + $(RM) lz4_all.c + test-install: lz4 lib liblz4.pc lz4_root=.. ./test_install.sh diff --git a/tests/checkFrame.c b/tests/checkFrame.c new file mode 100644 index 0000000..f7e7a50 --- /dev/null +++ b/tests/checkFrame.c @@ -0,0 +1,306 @@ + /* + checkFrame - verify frame headers + Copyright (C) Yann Collet 2014-2016 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - LZ4 homepage : http://www.lz4.org + - LZ4 source repository : https://github.com/lz4/lz4 + */ + + /*-************************************ + * Compiler specific + **************************************/ + #ifdef _MSC_VER /* Visual Studio */ + # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ + # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ + #endif + + + /*-************************************ + * Includes + **************************************/ + #include "util.h" /* U32 */ + #include <stdlib.h> /* malloc, free */ + #include <stdio.h> /* fprintf */ + #include <string.h> /* strcmp */ + #include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */ + #include <assert.h> + #include "lz4frame.h" /* include multiple times to test correctness/safety */ + #include "lz4frame.h" + #define LZ4F_STATIC_LINKING_ONLY + #include "lz4frame.h" + #include "lz4frame.h" + #include "lz4.h" /* LZ4_VERSION_STRING */ + #define XXH_STATIC_LINKING_ONLY + #include "xxhash.h" /* XXH64 */ + + + /*-************************************ + * Constants + **************************************/ + #define KB *(1U<<10) + #define MB *(1U<<20) + #define GB *(1U<<30) + + + /*-************************************ + * Macros + **************************************/ + #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) + #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } + + /************************************** + * Exceptions + ***************************************/ + #ifndef DEBUG + # define DEBUG 0 + #endif + #define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); + #define EXM_THROW(error, ...) \ +{ \ + DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAYLEVEL(1, "Error %i : ", error); \ + DISPLAYLEVEL(1, __VA_ARGS__); \ + DISPLAYLEVEL(1, " \n"); \ + return(error); \ +} + + + +/*-*************************************** +* Local Parameters +*****************************************/ +static U32 no_prompt = 0; +static U32 displayLevel = 2; +static U32 use_pause = 0; + + +/*-******************************************************* +* Fuzzer functions +*********************************************************/ +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +#define MAX(a,b) ( (a) > (b) ? (a) : (b) ) + +typedef struct { + void* srcBuffer; + size_t srcBufferSize; + void* dstBuffer; + size_t dstBufferSize; + LZ4F_decompressionContext_t ctx; +} cRess_t; + +static int createCResources(cRess_t *ress) +{ + ress->srcBufferSize = 4 MB; + ress->srcBuffer = malloc(ress->srcBufferSize); + ress->dstBufferSize = 4 MB; + ress->dstBuffer = malloc(ress->dstBufferSize); + + if (!ress->srcBuffer || !ress->dstBuffer) { + free(ress->srcBuffer); + free(ress->dstBuffer); + EXM_THROW(20, "Allocation error : not enough memory"); + } + + if (LZ4F_isError( LZ4F_createDecompressionContext(&(ress->ctx), LZ4F_VERSION) )) { + free(ress->srcBuffer); + free(ress->dstBuffer); + EXM_THROW(21, "Unable to create decompression context"); + } + return 0; +} + +static void freeCResources(cRess_t ress) +{ + free(ress.srcBuffer); + free(ress.dstBuffer); + + (void) LZ4F_freeDecompressionContext(ress.ctx); +} + +int frameCheck(cRess_t ress, FILE* const srcFile, unsigned bsid, size_t blockSize) +{ + LZ4F_errorCode_t nextToLoad = 0; + size_t curblocksize = 0; + int partialBlock = 0; + + /* Main Loop */ + for (;;) { + size_t readSize; + size_t pos = 0; + size_t decodedBytes = ress.dstBufferSize; + size_t remaining; + LZ4F_frameInfo_t frameInfo; + + /* Read input */ + readSize = fread(ress.srcBuffer, 1, ress.srcBufferSize, srcFile); + if (!readSize) break; /* reached end of file or stream */ + + while (pos < readSize) { /* still to read */ + /* Decode Input (at least partially) */ + if (!nextToLoad) { + /* LZ4F_decompress returned 0 : starting new frame */ + curblocksize = 0; + remaining = readSize - pos; + nextToLoad = LZ4F_getFrameInfo(ress.ctx, &frameInfo, (char*)(ress.srcBuffer)+pos, &remaining); + if (LZ4F_isError(nextToLoad)) EXM_THROW(22, "Error getting frame info: %s", LZ4F_getErrorName(nextToLoad)); /* XXX */ + if (frameInfo.blockSizeID != bsid) EXM_THROW(23, "Block size ID %u != expected %u", frameInfo.blockSizeID, bsid); + pos += remaining; + /* nextToLoad should be block header size */ + remaining = nextToLoad; + decodedBytes = ress.dstBufferSize; + nextToLoad = LZ4F_decompress(ress.ctx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, NULL); + if (LZ4F_isError(nextToLoad)) EXM_THROW(24, "Decompression error : %s", LZ4F_getErrorName(nextToLoad)); + pos += remaining; + } + decodedBytes = ress.dstBufferSize; + /* nextToLoad should be just enough to cover the next block */ + if (nextToLoad > (readSize - pos)) { + /* block is not fully contained in current buffer */ + partialBlock = 1; + remaining = readSize - pos; + } else { + if (partialBlock) { + partialBlock = 0; + } + remaining = nextToLoad; + } + nextToLoad = LZ4F_decompress(ress.ctx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, NULL); + if (LZ4F_isError(nextToLoad)) EXM_THROW(24, "Decompression error : %s", LZ4F_getErrorName(nextToLoad)); + curblocksize += decodedBytes; + pos += remaining; + if (!partialBlock) { + /* detect small block due to end of frame; the final 4-byte frame checksum could be left in the buffer */ + if ((curblocksize != 0) && (nextToLoad > 4)) { + if (curblocksize != blockSize) + EXM_THROW(25, "Block size %zu != expected %zu, pos %zu\n", curblocksize, blockSize, pos); + } + curblocksize = 0; + } + } + } + /* can be out because readSize == 0, which could be an fread() error */ + if (ferror(srcFile)) EXM_THROW(26, "Read error"); + + if (nextToLoad!=0) EXM_THROW(27, "Unfinished stream"); + + return 0; +} + +int FUZ_usage(const char* programName) +{ + DISPLAY( "Usage :\n"); + DISPLAY( " %s [args] filename\n", programName); + DISPLAY( "\n"); + DISPLAY( "Arguments :\n"); + DISPLAY( " -b# : expected blocksizeID [4-7] (required)\n"); + DISPLAY( " -B# : expected blocksize [32-4194304] (required)\n"); + DISPLAY( " -v : verbose\n"); + DISPLAY( " -h : display help and exit\n"); + return 0; +} + + +int main(int argc, const char** argv) +{ + int argNb; + int bsid=0; + size_t blockSize=0; + const char* const programName = argv[0]; + + /* Check command line */ + for (argNb=1; argNb<argc; argNb++) { + const char* argument = argv[argNb]; + + if(!argument) continue; /* Protection if argument empty */ + + /* Decode command (note : aggregated short commands are allowed) */ + if (argument[0]=='-') { + if (!strcmp(argument, "--no-prompt")) { + no_prompt=1; + displayLevel=1; + continue; + } + argument++; + + while (*argument!=0) { + switch(*argument) + { + case 'h': + return FUZ_usage(programName); + case 'v': + argument++; + displayLevel++; + break; + case 'q': + argument++; + displayLevel--; + break; + case 'p': /* pause at the end */ + argument++; + use_pause = 1; + break; + + case 'b': + argument++; + bsid=0; + while ((*argument>='0') && (*argument<='9')) { + bsid *= 10; + bsid += *argument - '0'; + argument++; + } + break; + + case 'B': + argument++; + blockSize=0; + while ((*argument>='0') && (*argument<='9')) { + blockSize *= 10; + blockSize += *argument - '0'; + argument++; + } + break; + + default: + ; + return FUZ_usage(programName); + } + } + } else { + int err; + FILE *srcFile; + cRess_t ress; + if (bsid == 0 || blockSize == 0) + return FUZ_usage(programName); + DISPLAY("Starting frame checker (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION_STRING); + err = createCResources(&ress); + if (err) return (err); + srcFile = fopen(argument, "rb"); + if ( srcFile==NULL ) { + freeCResources(ress); + EXM_THROW(1, "%s: %s \n", argument, strerror(errno)); + } + err = frameCheck(ress, srcFile, bsid, blockSize); + freeCResources(ress); + fclose(srcFile); + return (err); + } + } + return 0; +} diff --git a/tests/frametest.c b/tests/frametest.c index 4efeb6f..6d2cdd0 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -343,9 +343,10 @@ int basicTests(U32 seed, double compressibility) op += oSize; ip += iSize; } - { U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); - if (crcDest != crcOrig) goto _output_error; } - DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-ostart), COMPRESSIBLE_NOISE_LENGTH); + { U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); + if (crcDest != crcOrig) goto _output_error; + } + DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-ostart), (unsigned)COMPRESSIBLE_NOISE_LENGTH); } } @@ -520,7 +521,7 @@ int basicTests(U32 seed, double compressibility) LZ4F_CDict* const cdict = LZ4F_createCDict(CNBuffer, dictSize); if (cdict == NULL) goto _output_error; CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) ); - + DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with NULL dict : "); CHECK_V(cSizeNoDict, LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity, @@ -840,8 +841,8 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi size_t const iSize = MIN(sampleMax, (size_t)(iend-ip)); size_t const oSize = LZ4F_compressBound(iSize, prefsPtr); cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1); - DISPLAYLEVEL(6, "Sending %zi bytes to compress (stableSrc:%u) \n", - iSize, cOptions.stableSrc); + DISPLAYLEVEL(6, "Sending %u bytes to compress (stableSrc:%u) \n", + (unsigned)iSize, cOptions.stableSrc); result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions); CHECK(LZ4F_isError(result), "Compression failed (error %i : %s)", (int)result, LZ4F_getErrorName(result)); @@ -856,8 +857,18 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi } } } CHECK(op>=oend, "LZ4F_compressFrameBound overflow"); - result = LZ4F_compressEnd(cCtx, op, oend-op, &cOptions); - CHECK(LZ4F_isError(result), "Compression completion failed (error %i : %s)", (int)result, LZ4F_getErrorName(result)); + { size_t const dstEndSafeSize = LZ4F_compressBound(0, prefsPtr); + int const tooSmallDstEnd = ((FUZ_rand(&randState) & 31) == 3); + size_t const dstEndTooSmallSize = (FUZ_rand(&randState) % dstEndSafeSize) + 1; + size_t const dstEndSize = tooSmallDstEnd ? dstEndTooSmallSize : dstEndSafeSize; + BYTE const canaryByte = (BYTE)(FUZ_rand(&randState) & 255); + op[dstEndSize] = canaryByte; + result = LZ4F_compressEnd(cCtx, op, dstEndSize, &cOptions); + CHECK(op[dstEndSize] != canaryByte, "LZ4F_compressEnd writes beyond dstCapacity !"); + if (LZ4F_isError(result)) { + if (tooSmallDstEnd) /* failure is allowed */ continue; + CHECK(1, "Compression completion failed (error %i : %s)", (int)result, LZ4F_getErrorName(result)); + } } op += result; cSize = op-(BYTE*)compressedBuffer; DISPLAYLEVEL(5, "\nCompressed %u bytes into %u \n", (U32)srcSize, (U32)cSize); diff --git a/tests/fullbench.c b/tests/fullbench.c index fd1202d..4b4e921 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -367,7 +367,6 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) size_t readSize; int compressedBuffSize; U32 crcOriginal; - size_t errorCode; /* Check file existence */ if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } @@ -561,7 +560,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) case 8: decompressionFunction = local_LZ4_decompress_safe_forceExtDict; dName = "LZ4_decompress_safe_forceExtDict"; break; #endif case 9: decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress"; - errorCode = LZ4F_compressFrame(compressed_buff, compressedBuffSize, orig_buff, benchedSize, NULL); + { size_t const errorCode = LZ4F_compressFrame(compressed_buff, compressedBuffSize, orig_buff, benchedSize, NULL); if (LZ4F_isError(errorCode)) { DISPLAY("Error while preparing compressed frame\n"); free(orig_buff); @@ -573,6 +572,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) chunkP[0].compressedSize = (int)errorCode; nbChunks = 1; break; + } default : continue; /* skip if unknown ID */ } diff --git a/tests/fuzzer.c b/tests/fuzzer.c index b29e82e..1d23e1a 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -744,6 +744,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c LZ4_attach_dictionary(&LZ4_stream, &LZ4dict); blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue using extDictCtx failed"); + FUZ_CHECKTEST(LZ4_stream.internal_donotuse.dirty, "context should be good"); /* In the future, it might be desirable to let extDictCtx mode's * output diverge from the output generated by regular extDict mode. @@ -754,19 +755,21 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(XXH32(compressedBuffer, blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output"); FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary(), but output buffer is 1 byte too short"); - LZ4_resetStream(&LZ4_stream); + LZ4_resetStream_fast(&LZ4_stream); LZ4_attach_dictionary(&LZ4_stream, &LZ4dict); ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1); FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using extDictCtx should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize); + FUZ_CHECKTEST(!LZ4_stream.internal_donotuse.dirty, "context should be dirty"); FUZ_DISPLAYTEST(); - LZ4_resetStream(&LZ4_stream); + LZ4_resetStream_fast(&LZ4_stream); LZ4_attach_dictionary(&LZ4_stream, &LZ4dict); ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx should work : enough size available within output buffer"); FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output"); FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output"); + FUZ_CHECKTEST(LZ4_stream.internal_donotuse.dirty, "context should be good"); FUZ_DISPLAYTEST(); LZ4_resetStream_fast(&LZ4_stream); @@ -776,6 +779,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx with re-used context should work : enough size available within output buffer"); FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output"); FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output"); + FUZ_CHECKTEST(LZ4_stream.internal_donotuse.dirty, "context should be good"); } /* Decompress with dictionary as external */ @@ -828,17 +832,20 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c LZ4_setCompressionLevel(&LZ4dictHC, compressionLevel-1); blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue failed"); + FUZ_CHECKTEST(LZ4dictHC.internal_donotuse.dirty, "Context should be clean"); FUZ_DISPLAYTEST(); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDict should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize); + FUZ_CHECKTEST(!LZ4dictHC.internal_donotuse.dirty, "Context should be dirty"); FUZ_DISPLAYTEST(); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue should work : enough size available within output buffer"); + FUZ_CHECKTEST(LZ4dictHC.internal_donotuse.dirty, "Context should be clean"); FUZ_DISPLAYTEST(); decodedBuffer[blockSize] = 0; @@ -861,19 +868,22 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC); blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, (int)compressedBufferSize); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue with ExtDictCtx failed"); + FUZ_CHECKTEST(LZ4_streamHC.internal_donotuse.dirty, "Context should be clean"); FUZ_DISPLAYTEST(); - LZ4_resetStreamHC (&LZ4_streamHC, compressionLevel); + LZ4_resetStreamHC_fast (&LZ4_streamHC, compressionLevel); LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC); ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDictCtx should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize); + FUZ_CHECKTEST(!LZ4_streamHC.internal_donotuse.dirty, "Context should be dirty"); FUZ_DISPLAYTEST(); - LZ4_resetStreamHC (&LZ4_streamHC, compressionLevel); + LZ4_resetStreamHC_fast (&LZ4_streamHC, compressionLevel); LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC); ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue using ExtDictCtx should work : enough size available within output buffer"); + FUZ_CHECKTEST(LZ4_streamHC.internal_donotuse.dirty, "Context should be clean"); FUZ_DISPLAYTEST(); LZ4_resetStreamHC_fast (&LZ4_streamHC, compressionLevel); @@ -881,6 +891,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx and fast reset size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue using ExtDictCtx and fast reset should work : enough size available within output buffer"); + FUZ_CHECKTEST(LZ4_streamHC.internal_donotuse.dirty, "Context should be clean"); FUZ_DISPLAYTEST(); decodedBuffer[blockSize] = 0; @@ -990,6 +1001,7 @@ static void FUZ_unitTests(int compressionLevel) LZ4_resetStream(&streamingState); result = LZ4_compress_fast_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1, 1); FUZ_CHECKTEST(result==0, "LZ4_compress_fast_continue() compression failed!"); + FUZ_CHECKTEST(streamingState.internal_donotuse.dirty, "context should be clean") result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize); FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); @@ -1012,7 +1024,7 @@ static void FUZ_unitTests(int compressionLevel) XXH64_reset(&xxhOrig, 0); XXH64_reset(&xxhNewSafe, 0); XXH64_reset(&xxhNewFast, 0); - LZ4_resetStream(&streamingState); + LZ4_resetStream_fast(&streamingState); LZ4_setStreamDecode(&decodeStateSafe, NULL, 0); LZ4_setStreamDecode(&decodeStateFast, NULL, 0); @@ -1065,6 +1077,7 @@ static void FUZ_unitTests(int compressionLevel) LZ4_resetStreamHC(&sHC, compressionLevel); result = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize); FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); @@ -1073,10 +1086,11 @@ static void FUZ_unitTests(int compressionLevel) /* simple dictionary HC compression test */ crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, compressionLevel); + LZ4_resetStreamHC_fast(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, 64 KB); result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 64 KB); FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed"); @@ -1087,12 +1101,14 @@ static void FUZ_unitTests(int compressionLevel) { int result1, result2; int segSize = testCompressedSize / 2; crcOrig = XXH64(testInput + segSize, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, compressionLevel); + LZ4_resetStreamHC_fast(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, segSize); result1 = LZ4_compress_HC_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1); FUZ_CHECKTEST(result1==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result1); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); result2 = LZ4_compress_HC_continue(&sHC, testInput + 2*segSize, testCompressed+result1, segSize, segSize-1); FUZ_CHECKTEST(result2==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result2); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result1, segSize, testInput, segSize); FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 1 failed"); @@ -1104,10 +1120,11 @@ static void FUZ_unitTests(int compressionLevel) /* remote dictionary HC compression test */ crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, compressionLevel); + LZ4_resetStreamHC_fast(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, 32 KB); result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 32 KB); FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe_usingDict() decompression failed following remote dictionary HC compression test"); @@ -1125,7 +1142,7 @@ static void FUZ_unitTests(int compressionLevel) int segSize = (FUZ_rand(&randState) & 8191); int segNb = 1; - LZ4_resetStreamHC(&sHC, compressionLevel); + LZ4_resetStreamHC_fast(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, dict, dictSize); XXH64_reset(&crcOrigState, 0); @@ -1136,6 +1153,7 @@ static void FUZ_unitTests(int compressionLevel) crcOrig = XXH64_digest(&crcOrigState); result = LZ4_compress_HC_continue(&sHC, testInput + segStart, testCompressed, segSize, LZ4_compressBound(segSize)); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); result = LZ4_decompress_safe_usingDict(testCompressed, dst, result, segSize, dict, dictSize); FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %i failed", segNb); @@ -1172,7 +1190,7 @@ static void FUZ_unitTests(int compressionLevel) XXH64_reset(&xxhOrig, 0); XXH64_reset(&xxhNewSafe, 0); XXH64_reset(&xxhNewFast, 0); - LZ4_resetStreamHC(&sHC, compressionLevel); + LZ4_resetStreamHC_fast(&sHC, compressionLevel); LZ4_setStreamDecode(&decodeStateSafe, NULL, 0); LZ4_setStreamDecode(&decodeStateFast, NULL, 0); @@ -1183,6 +1201,7 @@ static void FUZ_unitTests(int compressionLevel) memcpy (ringBuffer + rNext, testInput + iNext, messageSize); compressedSize = LZ4_compress_HC_continue(&sHC, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed"); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, messageSize); FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe_continue() test failed"); @@ -1233,7 +1252,7 @@ static void FUZ_unitTests(int compressionLevel) XXH64_reset(&xxhOrig, 0); XXH64_reset(&xxhNewSafe, 0); XXH64_reset(&xxhNewFast, 0); - LZ4_resetStreamHC(&sHC, compressionLevel); + LZ4_resetStreamHC_fast(&sHC, compressionLevel); LZ4_setStreamDecode(&decodeStateSafe, NULL, 0); LZ4_setStreamDecode(&decodeStateFast, NULL, 0); @@ -1246,6 +1265,7 @@ static void FUZ_unitTests(int compressionLevel) compressedSize = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed"); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, ringBufferSafe + dNext, compressedSize, messageSize); FUZ_CHECKTEST(result!=messageSize, "64K D.ringBuffer : LZ4_decompress_safe_continue() test failed"); @@ -1277,6 +1297,7 @@ static void FUZ_unitTests(int compressionLevel) compressedSize = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed"); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); DISPLAYLEVEL(5, "compressed %i bytes to %i bytes \n", messageSize, compressedSize); /* test LZ4_decompress_safe_continue */ diff --git a/tests/test_custom_block_sizes.sh b/tests/test_custom_block_sizes.sh new file mode 100755 index 0000000..aba6733 --- /dev/null +++ b/tests/test_custom_block_sizes.sh @@ -0,0 +1,72 @@ +#/usr/bin/env sh +set -e + +LZ4=../lz4 +CHECKFRAME=./checkFrame +DATAGEN=./datagen + +failures="" + +TMPFILE=/tmp/test_custom_block_sizes.$$ +TMPFILE1=/tmp/test_custom_block_sizes1.$$ +TMPFILE2=/tmp/test_custom_block_sizes2.$$ +$DATAGEN -g12345678 > $TMPFILE1 +$DATAGEN -g12345678 > $TMPFILE2 + +echo Testing -B31 +$LZ4 -f -B31 $TMPFILE1 && failures="31 (should fail) " + +for blocksize in 32 65535 65536 +do + echo Testing -B$blocksize + $LZ4 -f -B$blocksize $TMPFILE1 + $LZ4 -f -B$blocksize $TMPFILE2 + cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4 + $CHECKFRAME -B$blocksize -b4 $TMPFILE.lz4 || failures="$failures $blocksize " +done + +for blocksize in 65537 262143 262144 +do + echo Testing -B$blocksize + $LZ4 -f -B$blocksize $TMPFILE1 + $LZ4 -f -B$blocksize $TMPFILE2 + cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4 + $CHECKFRAME -B$blocksize -b5 $TMPFILE.lz4 || failures="$failures $blocksize " +done + +for blocksize in 262145 1048575 1048576 +do + echo Testing -B$blocksize + $LZ4 -f -B$blocksize $TMPFILE1 + $LZ4 -f -B$blocksize $TMPFILE2 + cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4 + $CHECKFRAME -B$blocksize -b6 $TMPFILE.lz4 || failures="$failures $blocksize " +done + +for blocksize in 1048577 4194303 4194304 +do + echo Testing -B$blocksize + $LZ4 -f -B$blocksize $TMPFILE1 + $LZ4 -f -B$blocksize $TMPFILE2 + cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4 + $CHECKFRAME -B$blocksize -b7 $TMPFILE.lz4 || failures="$failures $blocksize " +done + +for blocksize in 4194305 10485760 +do + echo Testing -B$blocksize + $LZ4 -f -B$blocksize $TMPFILE1 + $LZ4 -f -B$blocksize $TMPFILE2 + cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4 + $CHECKFRAME -B4194304 -b7 $TMPFILE.lz4 || failures="$failures $blocksize " +done + +rm $TMPFILE.lz4 $TMPFILE1 $TMPFILE1.lz4 $TMPFILE2 $TMPFILE2.lz4 +if [ "$failures" == "" ] +then + echo ---- All tests passed + exit 0 +else + echo ---- The following tests had failures: $failures + exit 1 +fi |