diff options
author | Blaise Sanouillet <43636048+blezsan@users.noreply.github.com> | 2018-09-28 15:02:49 (GMT) |
---|---|---|
committer | Blaise Sanouillet <43636048+blezsan@users.noreply.github.com> | 2018-09-28 15:02:49 (GMT) |
commit | 64ad395d27ad4f935cbcf9a5ed21699fbf1ce89c (patch) | |
tree | f1c9c108be28773437ad5a4ae216967a28380a75 /tests | |
parent | cb917827f95a62dffb9575a2225e55721739b162 (diff) | |
download | lz4-64ad395d27ad4f935cbcf9a5ed21699fbf1ce89c.zip lz4-64ad395d27ad4f935cbcf9a5ed21699fbf1ce89c.tar.gz lz4-64ad395d27ad4f935cbcf9a5ed21699fbf1ce89c.tar.bz2 |
support custom block sizes
Diffstat (limited to 'tests')
-rw-r--r-- | tests/Makefile | 8 | ||||
-rw-r--r-- | tests/checkFrame.c | 269 | ||||
-rwxr-xr-x | tests/test_custom_block_sizes.sh | 72 |
3 files changed, 347 insertions, 2 deletions
diff --git a/tests/Makefile b/tests/Makefile index 3de111b..989ef85 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -63,7 +63,7 @@ NB_LOOPS ?= -i1 default: all -all: fullbench fuzzer frametest roundTripTest datagen +all: fullbench fuzzer frametest roundTripTest datagen checkFrame all32: CFLAGS+=-m32 all32: all @@ -109,6 +109,9 @@ roundTripTest : lz4.o lz4hc.o xxhash.o roundTripTest.c datagen : $(PRGDIR)/datagen.c datagencli.c $(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT) +checkFrame : lz4frame.o lz4.o lz4hc.o xxhash.o checkFrame.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + clean: @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) @$(MAKE) -C $(PRGDIR) $@ > $(VOID) @@ -118,7 +121,8 @@ clean: fuzzer$(EXT) fuzzer32$(EXT) \ frametest$(EXT) frametest32$(EXT) \ fasttest$(EXT) roundTripTest$(EXT) \ - datagen$(EXT) checkTag$(EXT) + datagen$(EXT) checkTag$(EXT) \ + frameTest$(EXT) @rm -fR $(TESTDIR) @echo Cleaning completed diff --git a/tests/checkFrame.c b/tests/checkFrame.c new file mode 100644 index 0000000..ace51c7 --- /dev/null +++ b/tests/checkFrame.c @@ -0,0 +1,269 @@ +/* + checkFrame - verify frame headers + Copyright (C) Yann Collet 2014-2016 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - LZ4 homepage : http://www.lz4.org + - LZ4 source repository : https://github.com/lz4/lz4 +*/ + +/*-************************************ +* Compiler specific +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ +#endif + + +/*-************************************ +* Includes +**************************************/ +#include "util.h" /* U32 */ +#include <stdlib.h> /* malloc, free */ +#include <stdio.h> /* fprintf */ +#include <string.h> /* strcmp */ +#include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */ +#include <assert.h> +#include "lz4frame.h" /* include multiple times to test correctness/safety */ +#include "lz4frame.h" +#define LZ4F_STATIC_LINKING_ONLY +#include "lz4frame.h" +#include "lz4frame.h" +#include "lz4.h" /* LZ4_VERSION_STRING */ +#define XXH_STATIC_LINKING_ONLY +#include "xxhash.h" /* XXH64 */ + + +/*-************************************ +* Constants +**************************************/ +#define KB *(1U<<10) +#define MB *(1U<<20) +#define GB *(1U<<30) + + +/*-************************************ +* Macros +**************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } + +/************************************** +* Exceptions +***************************************/ +#ifndef DEBUG +# define DEBUG 0 +#endif +#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); +#define EXM_THROW(error, ...) \ +{ \ + DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAYLEVEL(1, "Error %i : ", error); \ + DISPLAYLEVEL(1, __VA_ARGS__); \ + DISPLAYLEVEL(1, " \n"); \ + exit(error); \ +} + + + +/*-*************************************** +* Local Parameters +*****************************************/ +static U32 no_prompt = 0; +static U32 displayLevel = 2; +static U32 use_pause = 0; + + +/*-******************************************************* +* Fuzzer functions +*********************************************************/ +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +#define MAX(a,b) ( (a) > (b) ? (a) : (b) ) + +void frameCheck(const char *srcFileName, unsigned bsid, size_t blockSize) +{ + FILE *srcFile; + LZ4F_decompressionContext_t dctx = NULL; + size_t srcBufferSize = 4 MB; + void *srcBuffer = malloc(srcBufferSize); + size_t dstBufferSize = 4 MB; + void *dstBuffer = malloc(dstBufferSize); + LZ4F_errorCode_t nextToLoad = 0; + size_t curblocksize = 0; + int partialBlock = 0; + + if (!srcBuffer || !dstBuffer) EXM_THROW(31, "Allocation error : not enough memory"); + srcFile = fopen(srcFileName, "rb"); + if ( srcFile==NULL ) EXM_THROW(1, "%s: %s \n", srcFileName, strerror(errno)); + + if (LZ4F_isError( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) )) { + EXM_THROW(32, "Unable to create decompression context"); + } + /* Main Loop */ + for (;;) { + size_t readSize; + size_t pos = 0; + size_t decodedBytes = dstBufferSize; + size_t remaining; + LZ4F_frameInfo_t frameInfo; + + /* Read input */ + readSize = fread(srcBuffer, 1, srcBufferSize, srcFile); + if (!readSize) break; /* reached end of file or stream */ + + while (pos < readSize) { /* still to read */ + /* Decode Input (at least partially) */ + if (!nextToLoad) { + /* LZ4F_decompress returned 0 : starting new frame */ + curblocksize = 0; + remaining = readSize - pos; + nextToLoad = LZ4F_getFrameInfo(dctx, &frameInfo, (char*)(srcBuffer)+pos, &remaining); + if (LZ4F_isError(nextToLoad)) EXM_THROW(66, "Error getting frame info: %s", LZ4F_getErrorName(nextToLoad)); /* XXX */ + if (frameInfo.blockSizeID != bsid) EXM_THROW(66, "Block size ID %u != expected %u", frameInfo.blockSizeID, bsid); + pos += remaining; + /* nextToLoad should be block header size */ + remaining = nextToLoad; + decodedBytes = dstBufferSize; + nextToLoad = LZ4F_decompress(dctx, dstBuffer, &decodedBytes, (char*)(srcBuffer)+pos, &remaining, NULL); + if (LZ4F_isError(nextToLoad)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad)); + pos += remaining; + } + decodedBytes = dstBufferSize; + /* nextToLoad should be just enough to cover the next block */ + if (nextToLoad > (readSize - pos)) { + /* block is not fully contained in current buffer */ + partialBlock = 1; + remaining = readSize - pos; + } else { + if (partialBlock) { + partialBlock = 0; + } + remaining = nextToLoad; + } + nextToLoad = LZ4F_decompress(dctx, dstBuffer, &decodedBytes, (char*)(srcBuffer)+pos, &remaining, NULL); + if (LZ4F_isError(nextToLoad)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad)); + curblocksize += decodedBytes; + pos += remaining; + if (!partialBlock) { + /* detect small block due to end of frame; the final 4-byte frame checksum could be left in the buffer */ + if ((curblocksize != 0) && (nextToLoad > 4)) { + /* allow 4-byte fudge */ + if ((MAX(curblocksize, blockSize) - MIN(curblocksize, blockSize)) > 4) + EXM_THROW(66, "Block size %zu != expected %zu", curblocksize, blockSize); + } + curblocksize = 0; + } + } + } + /* can be out because readSize == 0, which could be an fread() error */ + if (ferror(srcFile)) EXM_THROW(67, "Read error"); + + if (nextToLoad!=0) EXM_THROW(68, "Unfinished stream"); + +} + +int FUZ_usage(const char* programName) +{ + DISPLAY( "Usage :\n"); + DISPLAY( " %s [args] filename\n", programName); + DISPLAY( "\n"); + DISPLAY( "Arguments :\n"); + DISPLAY( " -b# : expected blocksizeID [4-7] (required)\n"); + DISPLAY( " -B# : expected blocksize [4-7] (required)\n"); + DISPLAY( " -v : verbose\n"); + DISPLAY( " -h : display help and exit\n"); + return 0; +} + + +int main(int argc, const char** argv) +{ + int argNb; + int bsid=0; + size_t blockSize=0; + const char* const programName = argv[0]; + + /* Check command line */ + for (argNb=1; argNb<argc; argNb++) { + const char* argument = argv[argNb]; + + if(!argument) continue; /* Protection if argument empty */ + + /* Decode command (note : aggregated short commands are allowed) */ + if (argument[0]=='-') { + if (!strcmp(argument, "--no-prompt")) { + no_prompt=1; + displayLevel=1; + continue; + } + argument++; + + while (*argument!=0) { + switch(*argument) + { + case 'h': + return FUZ_usage(programName); + case 'v': + argument++; + displayLevel++; + break; + case 'q': + argument++; + displayLevel--; + break; + case 'p': /* pause at the end */ + argument++; + use_pause = 1; + break; + + case 'b': + argument++; + bsid=0; + while ((*argument>='0') && (*argument<='9')) { + bsid *= 10; + bsid += *argument - '0'; + argument++; + } + break; + + case 'B': + argument++; + blockSize=0; + while ((*argument>='0') && (*argument<='9')) { + blockSize *= 10; + blockSize += *argument - '0'; + argument++; + } + break; + + default: + ; + return FUZ_usage(programName); + } + } + } else { + if (bsid == 0 || blockSize == 0) + return FUZ_usage(programName); + DISPLAY("Starting frame checker (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION_STRING); + frameCheck(argument, bsid, blockSize); + } + } + return 0; +} diff --git a/tests/test_custom_block_sizes.sh b/tests/test_custom_block_sizes.sh new file mode 100755 index 0000000..7317b65 --- /dev/null +++ b/tests/test_custom_block_sizes.sh @@ -0,0 +1,72 @@ +#/usr/bin/env sh +set -e + +LZ4=../lz4 +CHECKFRAME=./checkFrame +DATAGEN=./datagen + +failures="" + +TMPFILE=/tmp/test_custom_block_sizes.$$ +TMPFILE1=/tmp/test_custom_block_sizes1.$$ +TMPFILE2=/tmp/test_custom_block_sizes2.$$ +$DATAGEN -g12345678 > $TMPFILE1 +$DATAGEN -g12345678 > $TMPFILE2 + +echo Testing -B32 +$LZ4 -f -B32 $TMPFILE && failures="31 (should fail) " + +for blocksize in 512 65535 65536 +do + echo Testing -B$blocksize + $LZ4 -f -B$blocksize $TMPFILE1 + $LZ4 -f -B$blocksize $TMPFILE2 + cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4 + $CHECKFRAME -B$blocksize -b4 $TMPFILE.lz4 || failures="$failures $blocksize " +done + +for blocksize in 65537 262143 262144 +do + echo Testing -B$blocksize + $LZ4 -f -B$blocksize $TMPFILE1 + $LZ4 -f -B$blocksize $TMPFILE2 + cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4 + $CHECKFRAME -B$blocksize -b5 $TMPFILE.lz4 || failures="$failures $blocksize " +done + +for blocksize in 262145 1048575 1048576 +do + echo Testing -B$blocksize + $LZ4 -f -B$blocksize $TMPFILE1 + $LZ4 -f -B$blocksize $TMPFILE2 + cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4 + $CHECKFRAME -B$blocksize -b6 $TMPFILE.lz4 || failures="$failures $blocksize " +done + +for blocksize in 1048577 4194303 4194304 +do + echo Testing -B$blocksize + $LZ4 -f -B$blocksize $TMPFILE1 + $LZ4 -f -B$blocksize $TMPFILE2 + cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4 + $CHECKFRAME -B$blocksize -b7 $TMPFILE.lz4 || failures="$failures $blocksize " +done + +for blocksize in 4194305 10485760 +do + echo Testing -B$blocksize + $LZ4 -f -B$blocksize $TMPFILE1 + $LZ4 -f -B$blocksize $TMPFILE2 + cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4 + $CHECKFRAME -B4194304 -b7 $TMPFILE.lz4 || failures="$failures $blocksize " +done + +rm $TMPFILE.lz4 $TMPFILE1 $TMPFILE1.lz4 $TMPFILE2 $TMPFILE2.lz4 +if [ "$failures" == "" ] +then + echo ---- All tests passed + exit 0 +else + echo ---- The following tests had failures: $failures + exit 1 +fi |