diff options
-rw-r--r-- | .travis.yml | 1 | ||||
-rw-r--r-- | Makefile | 2 | ||||
-rw-r--r-- | NEWS | 3 | ||||
-rw-r--r-- | lib/lz4.c | 2 | ||||
-rw-r--r-- | lib/lz4frame.c | 41 | ||||
-rw-r--r-- | programs/bench.c | 17 | ||||
-rw-r--r--[-rwxr-xr-x] | programs/datagen.c | 0 | ||||
-rw-r--r-- | programs/frametest.c | 9 | ||||
-rw-r--r-- | programs/fullbench.c | 27 | ||||
-rw-r--r-- | programs/fuzzer.c | 8 | ||||
-rw-r--r--[-rwxr-xr-x] | programs/lz4io.c | 18 |
11 files changed, 75 insertions, 53 deletions
diff --git a/.travis.yml b/.travis.yml index 650fba1..3ebda15 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ env: - LZ4_TRAVIS_CI_ENV=streaming-examples - LZ4_TRAVIS_CI_ENV=cmake - LZ4_TRAVIS_CI_ENV=gpptest + - LZ4_TRAVIS_CI_ENV=staticAnalyze - LZ4_TRAVIS_CI_ENV=dist - LZ4_TRAVIS_CI_ENV=test-lz4 - LZ4_TRAVIS_CI_ENV=test-lz4c @@ -130,6 +130,8 @@ cmake: gpptest: export CC=g++; export CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align"; $(MAKE) -e all +staticAnalyze: clean + export CFLAGS=-g; scan-build -v $(MAKE) all streaming-examples: cd examples; $(MAKE) -e test @@ -2,8 +2,9 @@ r128: New : lz4 cli sparse file support New : command -m, to compress multiple files in a single command Fixed : Restored lz4hc compression ratio (slightly lower since r124) -New : g++ compatibility test +New : g++ compatibility tests New : datagen can generate sparse files +New : scan-build tests Fixed : Fuzzer + frametest compatibility with NetBSD (issue #48) Added : Visual project directory @@ -1150,7 +1150,7 @@ typedef struct */ LZ4_streamDecode_t* LZ4_createStreamDecode(void) { - LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(LZ4_STREAMDECODESIZE_U64, sizeof(U64)); + LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t)); return lz4s; } diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 7153871..0c8dc53 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -45,8 +45,6 @@ Compiler Options # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ #endif -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - /************************************** * Memory routines @@ -639,6 +637,7 @@ size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, if (dstMaxSize < (cctxPtr->tmpInSize + 16)) return (size_t)-ERROR_dstMaxSize_tooSmall; memset(&cOptionsNull, 0, sizeof(cOptionsNull)); if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; + (void)compressOptionsPtr; /* not yet useful */ /* select compression function */ compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); @@ -805,11 +804,11 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const void* srcVo typedef enum { dstage_getHeader=0, dstage_storeHeader, dstage_decodeHeader, - dstage_getCBlockSize, dstage_storeCBlockSize, dstage_decodeCBlockSize, + dstage_getCBlockSize, dstage_storeCBlockSize, dstage_copyDirect, dstage_getCBlock, dstage_storeCBlock, dstage_decodeCBlock, dstage_decodeCBlock_intoDst, dstage_decodeCBlock_intoTmp, dstage_flushOut, - dstage_getSuffix, dstage_storeSuffix, dstage_checkSuffix + dstage_getSuffix, dstage_storeSuffix } dStage_t; @@ -950,7 +949,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* const dstEnd = dstStart + *dstSizePtr; BYTE* dstPtr = dstStart; - const BYTE* selectedIn=NULL; + const BYTE* selectedIn = NULL; unsigned doAnotherStage = 1; size_t nextSrcSizeHint = 1; @@ -1020,15 +1019,16 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, { selectedIn = srcPtr; srcPtr += 4; - dctxPtr->dStage = dstage_decodeCBlockSize; - break; } + else + { /* not enough input to read cBlockSize field */ - dctxPtr->tmpInSize = 0; - dctxPtr->dStage = dstage_storeCBlockSize; - break; + dctxPtr->tmpInSize = 0; + dctxPtr->dStage = dstage_storeCBlockSize; + } } + if (dctxPtr->dStage == dstage_storeCBlockSize) case dstage_storeCBlockSize: { size_t sizeToCopy = 4 - dctxPtr->tmpInSize; @@ -1043,11 +1043,9 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, break; } selectedIn = dctxPtr->tmpIn; - dctxPtr->dStage = dstage_decodeCBlockSize; - break; } - case dstage_decodeCBlockSize: + /* case dstage_decodeCBlockSize: */ /* no more direct access, to prevent scan-build warning */ { size_t nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU; if (nextCBlockSize==0) /* frameEnd signal, no more CBlock */ @@ -1235,18 +1233,19 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, doAnotherStage = 0; break; } - if ((srcEnd - srcPtr) >= 4) /* CRC present */ + if ((srcEnd - srcPtr) < 4) /* not enough size for entire CRC */ + { + dctxPtr->tmpInSize = 0; + dctxPtr->dStage = dstage_storeSuffix; + } + else { selectedIn = srcPtr; srcPtr += 4; - dctxPtr->dStage = dstage_checkSuffix; - break; } - dctxPtr->tmpInSize = 0; - dctxPtr->dStage = dstage_storeSuffix; - break; } + if (dctxPtr->dStage == dstage_storeSuffix) case dstage_storeSuffix: { size_t sizeToCopy = 4 - dctxPtr->tmpInSize; @@ -1261,11 +1260,9 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, break; } selectedIn = dctxPtr->tmpIn; - dctxPtr->dStage = dstage_checkSuffix; - break; } - case dstage_checkSuffix: + /* case dstage_checkSuffix: */ /* no direct call, to avoid scan-build warning */ { U32 readCRC = LZ4F_readLE32(selectedIn); U32 resultCRC = XXH32_digest(&(dctxPtr->xxh)); diff --git a/programs/bench.c b/programs/bench.c index b632314..f0a27ef 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -207,12 +207,17 @@ static size_t BMK_findMaxMem(U64 requiredMem) while (!testmem) { - requiredMem -= step; + if (requiredMem > step) requiredMem -= step; + else requiredMem >>= 1; testmem = (BYTE*) malloc ((size_t)requiredMem); } - free (testmem); - return (size_t) (requiredMem - step); + + /* keep some space available */ + if (requiredMem > step) requiredMem -= step; + else requiredMem >>= 1; + + return (size_t)requiredMem; } @@ -287,7 +292,9 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel) /* Memory allocation & restrictions */ inFileSize = BMK_GetFileSize(inFileName); + if (inFileSize==0) { DISPLAY( "file is empty\n"); return 11; } benchedSize = (size_t) BMK_findMaxMem(inFileSize * 2) / 2; + if (benchedSize==0) { DISPLAY( "not enough memory\n"); return 11; } if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; if (benchedSize < inFileSize) { @@ -296,11 +303,11 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel) /* Alloc */ chunkP = (struct chunkParameters*) malloc(((benchedSize / (size_t)chunkSize)+1) * sizeof(struct chunkParameters)); - orig_buff = (char*)malloc((size_t )benchedSize); + orig_buff = (char*)malloc((size_t)benchedSize); nbChunks = (int) ((int)benchedSize / chunkSize) + 1; maxCompressedChunkSize = LZ4_compressBound(chunkSize); compressedBuffSize = nbChunks * maxCompressedChunkSize; - compressedBuffer = (char*)malloc((size_t )compressedBuffSize); + compressedBuffer = (char*)malloc((size_t)compressedBuffSize); if (!orig_buff || !compressedBuffer) diff --git a/programs/datagen.c b/programs/datagen.c index bccb21e..bccb21e 100755..100644 --- a/programs/datagen.c +++ b/programs/datagen.c diff --git a/programs/frametest.c b/programs/frametest.c index df37e5f..c69de8c 100644 --- a/programs/frametest.c +++ b/programs/frametest.c @@ -263,6 +263,7 @@ int basicTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "Single Block : \n"); errorCode = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL); + if (LZ4F_isError(errorCode)) goto _output_error; crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); if (crcDest != crcOrig) goto _output_error; DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize); @@ -418,7 +419,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi # define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } - // Create buffers + /* Create buffers */ result = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result); result = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION); @@ -427,14 +428,14 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi CHECK(srcBuffer==NULL, "srcBuffer Allocation failed"); compressedBuffer = malloc(LZ4F_compressFrameBound(srcDataLength, NULL)); CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed"); - decodedBuffer = malloc(srcDataLength); + decodedBuffer = calloc(1, srcDataLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */ CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed"); FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand); - // jump to requested testNb + /* jump to requested testNb */ for (testNb =0; testNb < startTest; testNb++) (void)FUZ_rand(&coreRand); // sync randomizer - // main fuzzer loop + /* main fuzzer test loop */ for ( ; testNb < nbTests; testNb++) { U32 randState = coreRand ^ prime1; diff --git a/programs/fullbench.c b/programs/fullbench.c index 6ae28d7..017b020 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -114,9 +114,13 @@ #define NBLOOPS 6 #define TIMELOOP 2500 +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + #define KNUTH 2654435761U -#define MAX_MEM (1984<<20) -#define DEFAULT_CHUNKSIZE (4<<20) +#define MAX_MEM (1984 MB) +#define DEFAULT_CHUNKSIZE (4 MB) #define ALL_COMPRESSORS 0 #define ALL_DECOMPRESSORS 0 @@ -217,21 +221,26 @@ static int BMK_GetMilliSpan( int nTimeStart ) static size_t BMK_findMaxMem(U64 requiredMem) { - size_t step = (64U<<20); // 64 MB + size_t step = 64 MB; BYTE* testmem=NULL; - requiredMem = (((requiredMem >> 25) + 1) << 26); + requiredMem = (((requiredMem >> 26) + 1) << 26); + requiredMem += 2*step; if (requiredMem > MAX_MEM) requiredMem = MAX_MEM; - requiredMem += 2*step; while (!testmem) { - requiredMem -= step; + if (requiredMem > step) requiredMem -= step; + else requiredMem >>= 1; testmem = (BYTE*) malloc ((size_t)requiredMem); } - free (testmem); - return (size_t) (requiredMem - step); + + /* keep some space available */ + if (requiredMem > step) requiredMem -= step; + else requiredMem >>= 1; + + return (size_t)requiredMem; } @@ -566,7 +575,9 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) // Memory allocation & restrictions inFileSize = BMK_GetFileSize(inFileName); + if (inFileSize==0) { DISPLAY( "file is empty\n"); return 11; } benchedSize = (size_t) BMK_findMaxMem(inFileSize) / 2; + if (benchedSize==0) { DISPLAY( "not enough memory\n"); return 11; } if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; if (benchedSize < inFileSize) { diff --git a/programs/fuzzer.c b/programs/fuzzer.c index ee2beb8..d126da2 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -204,7 +204,7 @@ static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, dou #define BLOCKSIZE_I134 (32 MB) static int FUZ_AddressOverflow(void) { - char* buffers[MAX_NB_BUFF_I134+1] = {0}; + char* buffers[MAX_NB_BUFF_I134+1]; int i, nbBuff=0; int highAddress = 0; @@ -780,7 +780,7 @@ static void FUZ_unitTests(void) FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed"); XXH64_update(&xxhNew, testVerify + dNext, messageSize); - crcNew = crcOrig = XXH64_digest(&xxhNew); + crcNew = XXH64_digest(&xxhNew); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); // prepare next message @@ -943,7 +943,7 @@ static void FUZ_unitTests(void) FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed"); XXH64_update(&xxhNew, testVerify + dNext, messageSize); - crcNew = crcOrig = XXH64_digest(&xxhNew); + crcNew = XXH64_digest(&xxhNew); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); // prepare next message @@ -986,7 +986,7 @@ static void FUZ_unitTests(void) FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed"); XXH64_update(&xxhNew, testVerify + dNext, messageSize); - crcNew = crcOrig = XXH64_digest(&xxhNew); + crcNew = XXH64_digest(&xxhNew); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); // prepare next message diff --git a/programs/lz4io.c b/programs/lz4io.c index 94bf5ea..50eb035 100755..100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -63,7 +63,7 @@ # include <fcntl.h> /* _O_BINARY */ # include <io.h> /* _setmode, _fileno, _get_osfhandle */ # define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) -# include <Windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */
+# include <Windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ # define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } # if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ # define fseek _fseeki64 @@ -465,11 +465,12 @@ int LZ4IO_compressFilename(const char* input_filename, const char* output_filena } +#define FNSPACE 30 int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel) { int i; - char* outFileName = NULL; - size_t ofnSize = 0; + char* outFileName = (char*)malloc(FNSPACE); + size_t ofnSize = FNSPACE; const size_t suffixSize = strlen(suffix); for (i=0; i<ifntSize; i++) @@ -530,15 +531,16 @@ static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput) /* Read Block */ sizeCheck = fread(in_buff, 1, blockSize, finput); + if (sizeCheck!=blockSize) EXM_THROW(52, "Read error : cannot access compressed block !"); /* Decode Block */ decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE); - if (decodeSize < 0) EXM_THROW(52, "Decoding Failed ! Corrupted input detected !"); + if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !"); filesize += decodeSize; /* Write Block */ sizeCheck = fwrite(out_buff, 1, decodeSize, foutput); - if (sizeCheck != (size_t)decodeSize) EXM_THROW(53, "Write error : cannot write decoded block into output\n"); + if (sizeCheck != (size_t)decodeSize) EXM_THROW(54, "Write error : cannot write decoded block into output\n"); } /* Free */ @@ -721,9 +723,9 @@ int LZ4IO_decompressFilename(const char* input_filename, const char* output_file get_fileHandle(input_filename, output_filename, &finput, &foutput); /* sparse file */ - if (g_sparseFileSupport && foutput)
- {
- SET_SPARSE_FILE_MODE(foutput);
+ if (g_sparseFileSupport && foutput) + { + SET_SPARSE_FILE_MODE(foutput); } /* Loop over multiple streams */ |