From 2f8a4c32f998e6440a0b580996ecf8f101df2c74 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 9 Apr 2015 13:34:38 +0100 Subject: New LZ4_compress_safe() API --- lib/lz4.c | 39 ++++++++++++----------------- lib/lz4.h | 85 +++++++++++++++++++++++++++------------------------------------ 2 files changed, 53 insertions(+), 71 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index a06f711..9d7e5b6 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -739,14 +739,14 @@ _last_literals: } -int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize) +int LZ4_compress_safe_extState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize) { MEM_INIT(state, 0, LZ4_STREAMSIZE); if (maxOutputSize >= LZ4_compressBound(inputSize)) { if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, 1); + return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, 1); else return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, 1); } @@ -759,12 +759,7 @@ int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* } } -int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize) -{ - return LZ4_compress_limitedOutput_withState(state, source, dest, inputSize, LZ4_compressBound(inputSize)); -} - -int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) +int LZ4_compress_safe(const char* source, char* dest, int inputSize, int maxOutputSize) { #if (HEAPMODE) void* ctx = ALLOCATOR(LZ4_STREAMSIZE_U64, 8); /* malloc-calloc aligned on 8-bytes boundaries */ @@ -772,7 +767,7 @@ int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, in U64 ctx[LZ4_STREAMSIZE_U64] = {0}; /* Ensure data is aligned on 8-bytes boundaries */ #endif - int result = LZ4_compress_limitedOutput_withState(ctx, source, dest, inputSize, maxOutputSize); + int result = LZ4_compress_safe_extState(ctx, source, dest, inputSize, maxOutputSize); #if (HEAPMODE) FREEMEM(ctx); @@ -780,11 +775,6 @@ int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, in return result; } -int LZ4_compress(const char* source, char* dest, int inputSize) -{ - return LZ4_compress_limitedOutput(source, dest, inputSize, LZ4_compressBound(inputSize)); -} - int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, unsigned acceleration) { @@ -798,14 +788,14 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp if (acceleration == 0) { if (inputSize < LZ4_64Klimit) - result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, ACCELERATION_DEFAULT); + result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, ACCELERATION_DEFAULT); else result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, ACCELERATION_DEFAULT); } else { if (inputSize < LZ4_64Klimit) - result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); else result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); } @@ -899,7 +889,7 @@ static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) } -int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize) +int LZ4_compress_safe_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize) { LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_stream; const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; @@ -948,13 +938,8 @@ int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* s } } -int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) -{ - return LZ4_compress_limitedOutput_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize)); -} - -/* Hidden debug function, to force separate dictionary mode */ +/* Hidden debug function, to force external dictionary mode */ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) { LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_dict; @@ -1347,6 +1332,14 @@ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compres /*************************************************** * Obsolete Functions ***************************************************/ +/* obsolete compression functions */ +int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_safe(source, dest, inputSize, maxOutputSize); } +int LZ4_compress(const char* source, char* dest, int inputSize) { return LZ4_compress_safe(source, dest, inputSize, LZ4_compressBound(inputSize)); } +int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { return LZ4_compress_safe_extState(state, src, dst, srcSize, dstSize); } +int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_safe_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize)); } +int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_safe_continue(LZ4_stream, source, dest, inputSize, maxOutputSize); } +int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) { return LZ4_compress_safe_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize)); } + /* These function names are deprecated and should no longer be used. They are only provided here for compatibility with older user programs. diff --git a/lib/lz4.h b/lib/lz4.h index 81a11c9..659866b 100644 --- a/lib/lz4.h +++ b/lib/lz4.h @@ -70,18 +70,24 @@ int LZ4_versionNumber (void); * Simple Functions **************************************/ -int LZ4_compress (const char* source, char* dest, int sourceSize); +int LZ4_compress_safe (const char* source, char* dest, int sourceSize, int maxOutputSize); int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); /* -LZ4_compress() : - Compresses 'sourceSize' bytes from 'source' into 'dest'. - Destination buffer must be already allocated, - and must be sized to handle worst cases situations (input data not compressible) - Worst case size evaluation is provided by function LZ4_compressBound() - inputSize : Max supported value is LZ4_MAX_INPUT_SIZE - return : the number of bytes written in buffer dest - or 0 if the compression fails +LZ4_compress_limitedOutput() : + Compresses 'sourceSize' bytes from buffer 'source' + into already allocated 'dest' of size 'maxOutputSize'. + Compression runs faster when 'maxOutputSize' >= LZ4_compressBound(sourceSize). + That's because in such case, it is guaranteed to compress within 'dest' budget, even in worst case scenario. + Compressing into a more limited space budget requires additional checks. + If the function cannot compress 'source' into a limited 'dest' budget, + compression stops *immediately*, and result of the function is zero. + It greatly accelerates behavior on non-compressible input, but as a consequence, 'dest' content is not valid either. + This function never writes outside 'dest' buffer, nor read outside 'source' buffer. + sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE + maxOutputSize : full or partial size of buffer 'dest' (which must be already allocated) + return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) + or 0 if compression fails LZ4_decompress_safe() : compressedSize : is obviously the source size @@ -89,9 +95,8 @@ LZ4_decompress_safe() : return : the number of bytes decompressed into the destination buffer (necessarily <= maxDecompressedSize) If the destination buffer is not large enough, decoding will stop and output an error code (<0). If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function is protected against buffer overflow exploits, - and never writes outside of output buffer, nor reads outside of input buffer. - It is also protected against malicious data packets. + This function is protected against buffer overflow exploits, including malicious data packets. + It never writes outside of output buffer, nor reads outside of input buffer. */ @@ -113,22 +118,6 @@ LZ4_compressBound() : */ int LZ4_compressBound(int inputSize); - -/* -LZ4_compress_limitedOutput() : - Compress 'sourceSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'. - If it cannot achieve it, compression will stop, and result of the function will be zero. - This saves time and memory on detecting non-compressible (or barely compressible) data. - This function never writes outside of provided output buffer. - - sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE - maxOutputSize : is the size of the destination buffer (which must be already allocated) - return : the number of bytes written in buffer 'dest' - or 0 if compression fails -*/ -int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); - - /* LZ4_compress_fast() : Same as LZ4_compress_limitedOutput, but allows to select an "acceleration" factor. @@ -138,17 +127,15 @@ LZ4_compress_fast() : */ int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxOutputSize, unsigned acceleration); - /* -LZ4_compress_withState() : - Same compression functions, but using an externally allocated memory space to store compression state. +LZ4_compress_safe_withState() : + Same compression function, just using an externally allocated memory space to store compression state. Use LZ4_sizeofState() to know how much memory must be allocated, and then, provide it as 'void* state' to compression functions. - Note that 'state' must be aligned on 4-bytes boundaries. + Note that 'state' ptr must be aligned on 4-bytes boundaries. */ int LZ4_sizeofState(void); -int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); -int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +int LZ4_compress_safe_extState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); /* @@ -164,7 +151,6 @@ LZ4_decompress_fast() : */ int LZ4_decompress_fast (const char* source, char* dest, int originalSize); - /* LZ4_decompress_safe_partial() : This function decompress a compressed block of size 'compressedSize' at position 'source' @@ -220,19 +206,14 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_streamPtr); int LZ4_loadDict (LZ4_stream_t* LZ4_streamPtr, const char* dictionary, int dictSize); /* - * LZ4_compress_continue - * Compress data block 'source', using blocks compressed before as dictionary to improve compression ratio - * Previous data blocks are assumed to still be present at their previous location. - * dest buffer must be already allocated, and sized to at least LZ4_compressBound(inputSize) + * LZ4_compress_safe_continue + * Compress data block 'source', using data from previous blocks to improve compression ratio. + * Important : Previous data blocks are assumed to still be present and unmodified ! + * dest buffer must be already allocated. + * if maxOutpuSize >= (inputSize), compression is guaranteed to succeed. + * if not, and if target size objective cannot be met, compression stops, and function returns a zero. */ -int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); - -/* - * LZ4_compress_limitedOutput_continue - * Same as before, but also specify a maximum target compressed size (maxOutputSize) - * If objective cannot be met, compression exits, and returns a zero. - */ -int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); +int LZ4_compress_safe_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); /* * LZ4_saveDict @@ -299,6 +280,15 @@ int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalS /************************************** * Obsolete Functions **************************************/ + +/* Obsolete compression functions */ +int LZ4_compress (const char* source, char* dest, int sourceSize); +int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); +int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); +int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); +int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); + /* Obsolete decompression functions These function names are deprecated and should no longer be used. @@ -310,7 +300,6 @@ It is highly recommended to stop using these functions and migrate to newer ones /* int LZ4_uncompress (const char* source, char* dest, int outputSize); */ /* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */ - /* Obsolete streaming functions; use new streaming interface whenever possible */ void* LZ4_create (const char* inputBuffer); int LZ4_sizeofStreamState(void); -- cgit v0.12