diff options
| -rw-r--r-- | doc/lz4_manual.html | 62 | ||||
| -rw-r--r-- | doc/lz4frame_manual.html | 2 | ||||
| -rw-r--r-- | lib/lz4frame.c | 21 | ||||
| -rw-r--r-- | lib/lz4hc.c | 9 | ||||
| -rw-r--r-- | ossfuzz/compress_frame_fuzzer.c | 4 | ||||
| -rw-r--r-- | ossfuzz/fuzz_data_producer.c | 4 | ||||
| -rw-r--r-- | programs/lz4cli.c | 2 | ||||
| -rw-r--r-- | programs/lz4io.c | 22 | ||||
| -rw-r--r-- | programs/lz4io.h | 8 | ||||
| -rw-r--r-- | tests/checkFrame.c | 1 | ||||
| -rw-r--r-- | tests/fullbench.c | 24 |
11 files changed, 90 insertions, 69 deletions
diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html index a477584..edb90c7 100644 --- a/doc/lz4_manual.html +++ b/doc/lz4_manual.html @@ -117,7 +117,8 @@ The larger the acceleration value, the faster the algorithm, but also the lesser the compression. It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. An acceleration value of "1" is the same as regular LZ4_compress_default() - Values <= 0 will be replaced by ACCELERATION_DEFAULT (currently == 1, see lz4.c). + Values <= 0 will be replaced by LZ4_ACCELERATION_DEFAULT (currently == 1, see lz4.c). + Values > LZ4_ACCELERATION_MAX will be replaced by LZ4_ACCELERATION_MAX (currently == 65537, see lz4.c). </p></pre><BR> <pre><b>int LZ4_sizeofState(void); @@ -140,31 +141,53 @@ int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int src New value is necessarily <= input value. @return : Nb bytes written into 'dst' (necessarily <= targetDestSize) or 0 if compression fails. + + Note : from v1.8.2 to v1.9.1, this function had a bug (fixed un v1.9.2+): + the produced compressed content could, in specific circumstances, + require to be decompressed into a destination buffer larger + by at least 1 byte than the content to decompress. + If an application uses `LZ4_compress_destSize()`, + it's highly recommended to update liblz4 to v1.9.2 or better. + If this can't be done or ensured, + the receiving decompression function should provide + a dstCapacity which is > decompressedSize, by at least 1 byte. + See https://github.com/lz4/lz4/issues/859 for details + </p></pre><BR> <pre><b>int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity); </b><p> Decompress an LZ4 compressed block, of size 'srcSize' at position 'src', into destination buffer 'dst' of size 'dstCapacity'. Up to 'targetOutputSize' bytes will be decoded. - The function stops decoding on reaching this objective, - which can boost performance when only the beginning of a block is required. + The function stops decoding on reaching this objective. + This can be useful to boost performance + whenever only the beginning of a block is required. - @return : the number of bytes decoded in `dst` (necessarily <= dstCapacity) + @return : the number of bytes decoded in `dst` (necessarily <= targetOutputSize) If source stream is detected malformed, function returns a negative result. - Note : @return can be < targetOutputSize, if compressed block contains less data. + Note 1 : @return can be < targetOutputSize, if compressed block contains less data. + + Note 2 : targetOutputSize must be <= dstCapacity - Note 2 : this function features 2 parameters, targetOutputSize and dstCapacity, - and expects targetOutputSize <= dstCapacity. - It effectively stops decoding on reaching targetOutputSize, + Note 3 : this function effectively stops decoding on reaching targetOutputSize, so dstCapacity is kind of redundant. - This is because in a previous version of this function, - decoding operation would not "break" a sequence in the middle. - As a consequence, there was no guarantee that decoding would stop at exactly targetOutputSize, + This is because in older versions of this function, + decoding operation would still write complete sequences. + Therefore, there was no guarantee that it would stop writing at exactly targetOutputSize, it could write more bytes, though only up to dstCapacity. Some "margin" used to be required for this operation to work properly. - This is no longer necessary. - The function nonetheless keeps its signature, in an effort to not break API. + Thankfully, this is no longer necessary. + The function nonetheless keeps the same signature, in an effort to preserve API compatibility. + + Note 4 : If srcSize is the exact size of the block, + then targetOutputSize can be any value, + including larger than the block's decompressed size. + The function will, at most, generate block's decompressed size. + + Note 5 : If srcSize is _larger_ than block's compressed size, + then targetOutputSize **MUST** be <= block's decompressed size. + Otherwise, *silent corruption will occur*. </p></pre><BR> @@ -494,18 +517,17 @@ union LZ4_streamDecode_u { <pre><b>#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS # define LZ4_DEPRECATED(message) </b>/* disable deprecation warnings */<b> #else -# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) # if defined (__cplusplus) && (__cplusplus >= 201402) </b>/* C++14 or greater */<b> # define LZ4_DEPRECATED(message) [[deprecated(message)]] -# elif (LZ4_GCC_VERSION >= 405) || defined(__clang__) -# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) -# elif (LZ4_GCC_VERSION >= 301) -# define LZ4_DEPRECATED(message) __attribute__((deprecated)) # elif defined(_MSC_VER) # define LZ4_DEPRECATED(message) __declspec(deprecated(message)) +# elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 45)) +# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 31) +# define LZ4_DEPRECATED(message) __attribute__((deprecated)) # else -# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") -# define LZ4_DEPRECATED(message) +# pragma message("WARNING: LZ4_DEPRECATED needs custom implementation for this compiler") +# define LZ4_DEPRECATED(message) </b>/* disabled */<b> # endif #endif </b>/* LZ4_DISABLE_DEPRECATE_WARNINGS */<b> </b><p> diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html index 72f27c8..f94a647 100644 --- a/doc/lz4frame_manual.html +++ b/doc/lz4frame_manual.html @@ -167,7 +167,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); @return is always the same for a srcSize and prefsPtr. prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario. tech details : - @return includes the possibility that internal buffer might already be filled by up to (blockSize-1) bytes. + @return if automatic flushing is not enabled, includes the possibility that internal buffer might already be filled by up to (blockSize-1) bytes. It also includes frame footer (ending + checksum), since it might be generated by LZ4F_compressEnd(). @return doesn't include frame header, as it was already generated by LZ4F_compressBegin(). diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 890fc96..03facb0 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -533,7 +533,7 @@ void LZ4F_freeCDict(LZ4F_CDict* cdict) * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. * Object can release its memory using LZ4F_freeCompressionContext(); */ -LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version) +LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** LZ4F_compressionContextPtr, unsigned version) { LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)ALLOC_AND_ZERO(sizeof(LZ4F_cctx_t)); if (cctxPtr==NULL) return err0r(LZ4F_ERROR_allocation_failed); @@ -541,20 +541,18 @@ LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_c cctxPtr->version = version; cctxPtr->cStage = 0; /* Next stage : init stream */ - *LZ4F_compressionContextPtr = (LZ4F_compressionContext_t)cctxPtr; + *LZ4F_compressionContextPtr = cctxPtr; return LZ4F_OK_NoError; } -LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_compressionContext) +LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctxPtr) { - LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)LZ4F_compressionContext; - if (cctxPtr != NULL) { /* support free on NULL */ - FREEMEM(cctxPtr->lz4CtxPtr); /* works because LZ4_streamHC_t and LZ4_stream_t are simple POD types */ + FREEMEM(cctxPtr->lz4CtxPtr); /* note: LZ4_streamHC_t and LZ4_stream_t are simple POD types */ FREEMEM(cctxPtr->tmpBuff); - FREEMEM(LZ4F_compressionContext); + FREEMEM(cctxPtr); } return LZ4F_OK_NoError; @@ -1291,8 +1289,10 @@ static void LZ4F_updateDict(LZ4F_dctx* dctx, const BYTE* dstPtr, size_t dstSize, const BYTE* dstBufferStart, unsigned withinTmp) { - if (dctx->dictSize==0) + if (dctx->dictSize==0) { + assert(dstPtr != NULL); dctx->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */ + } if (dctx->dict + dctx->dictSize == dstPtr) { /* dictionary continuity, directly within dstBuffer */ dctx->dictSize += dstSize; @@ -1382,13 +1382,14 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, const BYTE* const srcEnd = srcStart + *srcSizePtr; const BYTE* srcPtr = srcStart; BYTE* const dstStart = (BYTE*)dstBuffer; - BYTE* const dstEnd = dstStart + *dstSizePtr; + BYTE* const dstEnd = dstStart ? dstStart + *dstSizePtr : NULL; BYTE* dstPtr = dstStart; const BYTE* selectedIn = NULL; unsigned doAnotherStage = 1; size_t nextSrcSizeHint = 1; + if (dstBuffer == NULL) assert(*dstSizePtr == 0); MEM_INIT(&optionsNull, 0, sizeof(optionsNull)); if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull; *srcSizePtr = 0; @@ -1590,7 +1591,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, selectedIn = srcPtr; srcPtr += dctx->tmpInTarget; - if (0) /* jump over next block */ + if (0) /* always jump over next block */ case dstage_storeCBlock: { size_t const wantedData = dctx->tmpInTarget - dctx->tmpInSize; size_t const inputLeft = (size_t)(srcEnd-srcPtr); diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 6e4d732..97dfabd 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -981,10 +981,10 @@ int LZ4_compress_HC_destSize(void* state, const char* source, char* dest, int* s /* allocation */ LZ4_streamHC_t* LZ4_createStreamHC(void) { - LZ4_streamHC_t* const LZ4_streamHCPtr = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t)); - if (LZ4_streamHCPtr==NULL) return NULL; - LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); /* full initialization, malloc'ed buffer can be full of garbage */ - return LZ4_streamHCPtr; + LZ4_streamHC_t* state = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t)); + if (state==NULL) return NULL; + state = LZ4_initStreamHC(state, sizeof(*state)); /* full initialization, malloc'ed buffer can be full of garbage */ + return state; } int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) @@ -1347,7 +1347,6 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1; /* Main Loop */ - assert(ip - anchor < LZ4_MAX_INPUT_SIZE); while (ip <= mflimit) { int const llen = (int)(ip - anchor); int best_mlen, best_off; diff --git a/ossfuzz/compress_frame_fuzzer.c b/ossfuzz/compress_frame_fuzzer.c index bb14fc2..568ae14 100644 --- a/ossfuzz/compress_frame_fuzzer.c +++ b/ossfuzz/compress_frame_fuzzer.c @@ -28,8 +28,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) char* const dst = (char*)malloc(dstCapacity); char* const rt = (char*)malloc(size); - FUZZ_ASSERT(dst); - FUZZ_ASSERT(rt); + FUZZ_ASSERT(dst!=NULL); + FUZZ_ASSERT(rt!=NULL); /* If compression succeeds it must round trip correctly. */ size_t const dstSize = diff --git a/ossfuzz/fuzz_data_producer.c b/ossfuzz/fuzz_data_producer.c index cc06958..670fbf5 100644 --- a/ossfuzz/fuzz_data_producer.c +++ b/ossfuzz/fuzz_data_producer.c @@ -5,8 +5,8 @@ struct FUZZ_dataProducer_s{ size_t size; }; -FUZZ_dataProducer_t *FUZZ_dataProducer_create(const uint8_t *data, size_t size) { - FUZZ_dataProducer_t *producer = malloc(sizeof(FUZZ_dataProducer_t)); +FUZZ_dataProducer_t* FUZZ_dataProducer_create(const uint8_t* data, size_t size) { + FUZZ_dataProducer_t* const producer = malloc(sizeof(FUZZ_dataProducer_t)); FUZZ_ASSERT(producer != NULL); diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 67dcaa1..d7d4f81 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -395,7 +395,7 @@ int main(int argc, const char** argv) if (!strcmp(argument, "--favor-decSpeed")) { LZ4IO_favorDecSpeed(prefs, 1); continue; } if (!strcmp(argument, "--verbose")) { displayLevel++; continue; } if (!strcmp(argument, "--quiet")) { if (displayLevel) displayLevel--; continue; } - if (!strcmp(argument, "--version")) { DISPLAYOUT(WELCOME_MESSAGE); return 0; } + if (!strcmp(argument, "--version")) { DISPLAYOUT(WELCOME_MESSAGE); goto _cleanup; } if (!strcmp(argument, "--help")) { usage_advanced(exeName); goto _cleanup; } if (!strcmp(argument, "--keep")) { LZ4IO_setRemoveSrcFile(prefs, 0); continue; } /* keep source file (default) */ if (!strcmp(argument, "--rm")) { LZ4IO_setRemoveSrcFile(prefs, 1); continue; } diff --git a/programs/lz4io.c b/programs/lz4io.c index 0f3507e..3ffc519 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -1291,7 +1291,7 @@ int LZ4IO_decompressMultipleFilenames(LZ4IO_prefs_t* const prefs, size_t const suffixSize = strlen(suffix); dRess_t ress = LZ4IO_createDResources(prefs); - if (outFileName==NULL) return ifntSize; /* not enough memory */ + if (outFileName==NULL) EXM_THROW(70, "Memory allocation error"); ress.dstFile = LZ4IO_openDstFile(prefs, stdoutmark); for (i=0; i<ifntSize; i++) { @@ -1305,7 +1305,7 @@ int LZ4IO_decompressMultipleFilenames(LZ4IO_prefs_t* const prefs, free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); - if (outFileName==NULL) return ifntSize; + if (outFileName==NULL) EXM_THROW(71, "Memory allocation error"); } if (ifnSize <= suffixSize || strcmp(suffixPtr, suffix) != 0) { DISPLAYLEVEL(1, "File extension doesn't match expected LZ4_EXTENSION (%4s); will not process file: %s\n", suffix, inFileNamesTable[i]); @@ -1458,7 +1458,7 @@ static const char* LZ4IO_baseName(const char* input_filename) { const char* b = strrchr(input_filename, '/'); if (!b) b = strrchr(input_filename, '\\'); if (!b) return input_filename; - return b ? b + 1 : b; + return b + 1; } /* Report frame/s information in verbose mode. @@ -1472,7 +1472,9 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam LZ4IO_infoResult result = LZ4IO_format_not_known; /* default result (error) */ unsigned char buffer[LZ4F_HEADER_SIZE_MAX]; FILE* const finput = LZ4IO_openSrcFile(input_filename); - cfinfo->fileSize = (finput == NULL) ? 0 : UTIL_getOpenFileSize(finput); + + if (finput == NULL) return LZ4IO_not_a_file; + cfinfo->fileSize = UTIL_getOpenFileSize(finput); while (!feof(finput)) { LZ4IO_frameInfo_t frameInfo = LZ4IO_INIT_FRAMEINFO; @@ -1560,8 +1562,7 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam totalBlocksSize + 4, "-", "-"); result = LZ4IO_LZ4F_OK; - } - } + } } break; case LZ4IO_SKIPPABLE0: frameInfo.frameType = skippableFrame; @@ -1628,8 +1629,7 @@ int LZ4IO_displayCompressedFilesInfo(const char** inFileNames, size_t ifnIdx) assert(op_result == LZ4IO_format_not_known); DISPLAYLEVEL(1, "lz4: %s: File format not recognized \n", inFileNames[idx]); return 0; - } - } + } } DISPLAYLEVEL(3, "\n"); if (g_displayLevel < 3) { /* Display Summary */ @@ -1648,10 +1648,8 @@ int LZ4IO_displayCompressedFilesInfo(const char** inFileNames, size_t ifnIdx) DISPLAYOUT("%9s %s\n", "-", cfinfo.fileName); - } - } - } - } + } } } /* if (g_displayLevel < 3) */ + } /* for (; idx < ifnIdx; idx++) */ return result; } diff --git a/programs/lz4io.h b/programs/lz4io.h index b189e35..4763180 100644 --- a/programs/lz4io.h +++ b/programs/lz4io.h @@ -39,13 +39,13 @@ /* ************************************************** */ /* Special input/output values */ /* ************************************************** */ +#define stdinmark "stdin" +#define stdoutmark "stdout" #define NULL_OUTPUT "null" -static const char stdinmark[] = "stdin"; -static const char stdoutmark[] = "stdout"; #ifdef _WIN32 -static const char nulmark[] = "nul"; +#define nulmark "nul" #else -static const char nulmark[] = "/dev/null"; +#define nulmark "/dev/null" #endif /* ************************************************** */ diff --git a/tests/checkFrame.c b/tests/checkFrame.c index 139a599..f39d2ac 100644 --- a/tests/checkFrame.c +++ b/tests/checkFrame.c @@ -301,6 +301,7 @@ int main(int argc, const char** argv) freeCResources(ress); EXM_THROW(1, "%s: %s \n", argument, strerror(errno)); } + assert (srcFile != NULL); err = frameCheck(ress, srcFile, bsid, blockSize); freeCResources(ress); fclose(srcFile); diff --git a/tests/fullbench.c b/tests/fullbench.c index 7d74d3f..77f475b 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -399,25 +399,25 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) char* compressed_buff=NULL; const char* const inFileName = fileNamesTable[fileIdx++]; FILE* const inFile = fopen( inFileName, "rb" ); - U64 inFileSize; - size_t benchedSize; + U64 const inFileSize = UTIL_getFileSize(inFileName); + size_t benchedSize = BMK_findMaxMem(inFileSize*2) / 2; /* because 2 buffers */ int nbChunks; int maxCompressedChunkSize; size_t readSize; int compressedBuffSize; U32 crcOriginal; - /* Check file existence */ - if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } + /* Check infile pre-requisites */ + if (inFile==NULL) { DISPLAY("Pb opening %s \n", inFileName); return 11; } + if (inFileSize==0) { DISPLAY("file is empty \n"); fclose(inFile); return 11; } + if (benchedSize==0) { DISPLAY("not enough memory \n"); fclose(inFile); return 11; } /* Memory size adjustments */ - inFileSize = UTIL_getFileSize(inFileName); - if (inFileSize==0) { DISPLAY( "file is empty\n"); fclose(inFile); return 11; } - benchedSize = BMK_findMaxMem(inFileSize*2) / 2; /* because 2 buffers */ - if (benchedSize==0) { DISPLAY( "not enough memory\n"); fclose(inFile); return 11; } if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; - if (benchedSize < inFileSize) - DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); + if (benchedSize < inFileSize) { + DISPLAY("Not enough memory for '%s' full size; testing %i MB only... \n", + inFileName, (int)(benchedSize>>20)); + } /* Allocation */ chunkP = (struct chunkParameters*) malloc(((benchedSize / (size_t)g_chunkSize)+1) * sizeof(struct chunkParameters)); @@ -427,7 +427,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) compressedBuffSize = nbChunks * maxCompressedChunkSize; compressed_buff = (char*)malloc((size_t)compressedBuffSize); if(!chunkP || !orig_buff || !compressed_buff) { - DISPLAY("\nError: not enough memory!\n"); + DISPLAY("\nError: not enough memory! \n"); fclose(inFile); free(orig_buff); free(compressed_buff); @@ -475,7 +475,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) size_t remaining = benchedSize; char* in = orig_buff; char* out = compressed_buff; - nbChunks = (int) (((int)benchedSize + (g_chunkSize-1))/ g_chunkSize); + assert(nbChunks >= 1); for (i=0; i<nbChunks; i++) { chunkP[i].id = (U32)i; chunkP[i].origBuffer = in; in += g_chunkSize; |