diff options
author | Yann Collet <Cyan4973@users.noreply.github.com> | 2022-07-29 18:56:22 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-07-29 18:56:22 (GMT) |
commit | 5c8bb8739eb5e503a02a1976d1ddcadb049a041d (patch) | |
tree | 0cfdde4daa06923e62f050802434f1e5f4b7fe98 | |
parent | fb46a7fe8cd99bb9b8d9fc95d8fa6c238593a740 (diff) | |
parent | 4f4d09a0d1604da4a93d05716e586d73231c2e63 (diff) | |
download | lz4-5c8bb8739eb5e503a02a1976d1ddcadb049a041d.zip lz4-5c8bb8739eb5e503a02a1976d1ddcadb049a041d.tar.gz lz4-5c8bb8739eb5e503a02a1976d1ddcadb049a041d.tar.bz2 |
Merge pull request #1121 from lz4/decBench
Implement decoder-only benchmark
-rw-r--r-- | lib/lz4.h | 21 | ||||
-rw-r--r-- | lib/lz4frame.h | 75 | ||||
-rw-r--r-- | programs/bench.c | 426 | ||||
-rw-r--r-- | programs/bench.h | 26 | ||||
-rw-r--r-- | programs/lz4cli.c | 13 | ||||
-rw-r--r-- | tests/Makefile | 3 | ||||
-rw-r--r-- | tests/fullbench.c | 3 |
7 files changed, 344 insertions, 223 deletions
@@ -417,7 +417,10 @@ LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize); * save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression, * then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block. */ -LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity); +LZ4LIB_API int +LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, + const char* src, char* dst, + int srcSize, int dstCapacity); /*! LZ4_decompress_*_usingDict() : @@ -428,9 +431,17 @@ LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecod * Performance tip : Decompression speed can be substantially increased * when dst == dictStart + dictSize. */ -LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize); +LZ4LIB_API int +LZ4_decompress_safe_usingDict(const char* src, char* dst, + int srcSize, int dstCapacity, + const char* dictStart, int dictSize); + +LZ4LIB_API int +LZ4_decompress_safe_partial_usingDict(const char* src, char* dst, + int compressedSize, + int targetOutputSize, int maxOutputSize, + const char* dictStart, int dictSize); -LZ4LIB_API int LZ4_decompress_safe_partial_usingDict(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxOutputSize, const char* dictStart, int dictSize); #endif /* LZ4_H_2983827168210 */ @@ -508,7 +519,9 @@ LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const c * stream (and source buffer) must remain in-place / accessible / unchanged * through the completion of the first compression call on the stream. */ -LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream); +LZ4LIB_STATIC_API void +LZ4_attach_dictionary(LZ4_stream_t* workingStream, + const LZ4_stream_t* dictionaryStream); /*! In-place compression and decompression diff --git a/lib/lz4frame.h b/lib/lz4frame.h index a864ef9..1c2d0dd 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -54,12 +54,12 @@ extern "C" { /** - Introduction - - lz4frame.h implements LZ4 frame specification (doc/lz4_Frame_format.md). - lz4frame.h provides frame compression functions that take care - of encoding standard metadata alongside LZ4-compressed blocks. -*/ + * Introduction + * + * lz4frame.h implements LZ4 frame specification: see doc/lz4_Frame_format.md . + * LZ4 Frames are compatible with `lz4` CLI, + * and designed to be interoperable with any system. +**/ /*-*************************************************************** * Compiler specifics @@ -397,7 +397,7 @@ LZ4FLIB_API size_t LZ4F_headerSize(const void* src, size_t srcSize); /*! LZ4F_getFrameInfo() : * This function extracts frame parameters (max blockSize, dictID, etc.). - * Its usage is optional: user can call LZ4F_decompress() directly. + * Its usage is optional: user can also invoke LZ4F_decompress() directly. * * Extracted information will fill an existing LZ4F_frameInfo_t structure. * This can be useful for allocation and dictionary identification purposes. @@ -438,9 +438,10 @@ LZ4FLIB_API size_t LZ4F_headerSize(const void* src, size_t srcSize); * note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely. * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure. */ -LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, - LZ4F_frameInfo_t* frameInfoPtr, - const void* srcBuffer, size_t* srcSizePtr); +LZ4FLIB_API size_t +LZ4F_getFrameInfo(LZ4F_dctx* dctx, + LZ4F_frameInfo_t* frameInfoPtr, + const void* srcBuffer, size_t* srcSizePtr); /*! LZ4F_decompress() : * Call this function repetitively to regenerate data compressed in `srcBuffer`. @@ -473,10 +474,11 @@ LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, * * After a frame is fully decoded, dctx can be used again to decompress another frame. */ -LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx, - void* dstBuffer, size_t* dstSizePtr, - const void* srcBuffer, size_t* srcSizePtr, - const LZ4F_decompressOptions_t* dOptPtr); +LZ4FLIB_API size_t +LZ4F_decompress(LZ4F_dctx* dctx, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const LZ4F_decompressOptions_t* dOptPtr); /*! LZ4F_resetDecompressionContext() : added in v1.8.0 @@ -572,10 +574,11 @@ LZ4FLIB_STATIC_API size_t LZ4F_getBlockSize(LZ4F_blockSizeID_t blockSizeID); * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). * or an error code if it fails (which can be tested using LZ4F_isError()) */ -LZ4FLIB_STATIC_API size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctx, - void* dstBuffer, size_t dstCapacity, - const void* srcBuffer, size_t srcSize, - const LZ4F_compressOptions_t* cOptPtr); +LZ4FLIB_STATIC_API size_t +LZ4F_uncompressedUpdate(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_compressOptions_t* cOptPtr); /********************************** * Bulk processing dictionary API @@ -619,12 +622,12 @@ LZ4FLIB_STATIC_API void LZ4F_freeCDict(LZ4F_CDict* CDict); * but it's not recommended, as it's the only way to provide dictID in the frame header. * @return : number of bytes written into dstBuffer. * or an error code if it fails (can be tested using LZ4F_isError()) */ -LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict( - LZ4F_cctx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const LZ4F_CDict* cdict, - const LZ4F_preferences_t* preferencesPtr); +LZ4FLIB_STATIC_API size_t +LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* preferencesPtr); /*! LZ4F_compressBegin_usingCDict() : @@ -634,23 +637,23 @@ LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict( * however, it's the only way to provide dictID in the frame header. * @return : number of bytes written into dstBuffer for the header, * or an error code (which can be tested using LZ4F_isError()) */ -LZ4FLIB_STATIC_API size_t LZ4F_compressBegin_usingCDict( - LZ4F_cctx* cctx, - void* dstBuffer, size_t dstCapacity, - const LZ4F_CDict* cdict, - const LZ4F_preferences_t* prefsPtr); +LZ4FLIB_STATIC_API size_t +LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* prefsPtr); /*! LZ4F_decompress_usingDict() : * Same as LZ4F_decompress(), using a predefined dictionary. * Dictionary is used "in place", without any preprocessing. - * It must remain accessible throughout the entire frame decoding. */ -LZ4FLIB_STATIC_API size_t LZ4F_decompress_usingDict( - LZ4F_dctx* dctxPtr, - void* dstBuffer, size_t* dstSizePtr, - const void* srcBuffer, size_t* srcSizePtr, - const void* dict, size_t dictSize, - const LZ4F_decompressOptions_t* decompressOptionsPtr); +** It must remain accessible throughout the entire frame decoding. */ +LZ4FLIB_STATIC_API size_t +LZ4F_decompress_usingDict(LZ4F_dctx* dctxPtr, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const void* dict, size_t dictSize, + const LZ4F_decompressOptions_t* decompressOptionsPtr); /*! Custom memory allocation : diff --git a/programs/bench.c b/programs/bench.c index 603705c..7836717 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -51,10 +51,98 @@ #include "lz4.h" #define LZ4_HC_STATIC_LINKING_ONLY #include "lz4hc.h" +#include "lz4frame.h" /* LZ4F_decompress */ /* ************************************* -* Compression parameters and functions +* Constants +***************************************/ +#ifndef LZ4_GIT_COMMIT_STRING +# define LZ4_GIT_COMMIT_STRING "" +#else +# define LZ4_GIT_COMMIT_STRING LZ4_EXPAND_AND_QUOTE(LZ4_GIT_COMMIT) +#endif + +#define NBSECONDS 3 +#define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */ +#define TIMELOOP_NANOSEC 1*1000000000ULL /* 1 second */ +#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */ +#define COOLPERIOD_SEC 10 +#define DECOMP_MULT 1 /* test decompression DECOMP_MULT times longer than compression */ + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define LZ4_MAX_DICT_SIZE (64 KB) + +static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); + +static U32 g_compressibilityDefault = 50; + + +/* ************************************* +* console display +***************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ + +#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ + if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \ + { g_time = clock(); DISPLAY(__VA_ARGS__); \ + if (g_displayLevel>=4) fflush(stdout); } } +static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; +static clock_t g_time = 0; + + +/* ************************************* +* DEBUG and error conditions +***************************************/ +#ifndef DEBUG +# define DEBUG 0 +#endif +#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); +#define END_PROCESS(error, ...) \ +{ \ + DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAYLEVEL(1, "Error %i : ", error); \ + DISPLAYLEVEL(1, __VA_ARGS__); \ + DISPLAYLEVEL(1, "\n"); \ + exit(error); \ +} + +#define LZ4_isError(errcode) (errcode==0) + + +/* ************************************* +* Benchmark Parameters +***************************************/ +static U32 g_nbSeconds = NBSECONDS; +static size_t g_blockSize = 0; +int g_additionalParam = 0; +int g_benchSeparately = 0; +int g_decodeOnly = 0; + +void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; } + +void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; } + +void BMK_setNbSeconds(unsigned nbSeconds) +{ + g_nbSeconds = nbSeconds; + DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbSeconds); +} + +void BMK_setBlockSize(size_t blockSize) { g_blockSize = blockSize; } + +void BMK_setBenchSeparately(int separate) { g_benchSeparately = (separate!=0); } + +void BMK_setDecodeOnlyMode(int set) { g_decodeOnly = (set!=0); } + + +/* ************************************* + * Compression state management ***************************************/ struct compressionParameters @@ -79,8 +167,8 @@ struct compressionParameters const struct compressionParameters* pThis); }; -static void LZ4_compressInitNoStream( - struct compressionParameters* pThis) +static void +LZ4_compressInitNoStream(struct compressionParameters* pThis) { pThis->LZ4_stream = NULL; pThis->LZ4_dictStream = NULL; @@ -88,8 +176,8 @@ static void LZ4_compressInitNoStream( pThis->LZ4_dictStreamHC = NULL; } -static void LZ4_compressInitStream( - struct compressionParameters* pThis) +static void +LZ4_compressInitStream(struct compressionParameters* pThis) { pThis->LZ4_stream = LZ4_createStream(); pThis->LZ4_dictStream = LZ4_createStream(); @@ -98,8 +186,8 @@ static void LZ4_compressInitStream( LZ4_loadDict(pThis->LZ4_dictStream, pThis->dictBuf, pThis->dictSize); } -static void LZ4_compressInitStreamHC( - struct compressionParameters* pThis) +static void +LZ4_compressInitStreamHC(struct compressionParameters* pThis) { pThis->LZ4_stream = NULL; pThis->LZ4_dictStream = NULL; @@ -108,83 +196,84 @@ static void LZ4_compressInitStreamHC( LZ4_loadDictHC(pThis->LZ4_dictStreamHC, pThis->dictBuf, pThis->dictSize); } -static void LZ4_compressResetNoStream( - const struct compressionParameters* pThis) +static void +LZ4_compressResetNoStream(const struct compressionParameters* pThis) { (void)pThis; } -static void LZ4_compressResetStream( - const struct compressionParameters* pThis) +static void +LZ4_compressResetStream(const struct compressionParameters* pThis) { LZ4_resetStream_fast(pThis->LZ4_stream); LZ4_attach_dictionary(pThis->LZ4_stream, pThis->LZ4_dictStream); } -static void LZ4_compressResetStreamHC( - const struct compressionParameters* pThis) +static void +LZ4_compressResetStreamHC(const struct compressionParameters* pThis) { LZ4_resetStreamHC_fast(pThis->LZ4_streamHC, pThis->cLevel); LZ4_attach_HC_dictionary(pThis->LZ4_streamHC, pThis->LZ4_dictStreamHC); } -static int LZ4_compressBlockNoStream( - const struct compressionParameters* pThis, - const char* src, char* dst, - int srcSize, int dstSize) +static int +LZ4_compressBlockNoStream(const struct compressionParameters* pThis, + const char* src, char* dst, + int srcSize, int dstSize) { int const acceleration = (pThis->cLevel < 0) ? -pThis->cLevel + 1 : 1; return LZ4_compress_fast(src, dst, srcSize, dstSize, acceleration); } -static int LZ4_compressBlockNoStreamHC( - const struct compressionParameters* pThis, - const char* src, char* dst, - int srcSize, int dstSize) +static int +LZ4_compressBlockNoStreamHC(const struct compressionParameters* pThis, + const char* src, char* dst, + int srcSize, int dstSize) { return LZ4_compress_HC(src, dst, srcSize, dstSize, pThis->cLevel); } -static int LZ4_compressBlockStream( - const struct compressionParameters* pThis, - const char* src, char* dst, - int srcSize, int dstSize) +static int +LZ4_compressBlockStream(const struct compressionParameters* pThis, + const char* src, char* dst, + int srcSize, int dstSize) { int const acceleration = (pThis->cLevel < 0) ? -pThis->cLevel + 1 : 1; return LZ4_compress_fast_continue(pThis->LZ4_stream, src, dst, srcSize, dstSize, acceleration); } -static int LZ4_compressBlockStreamHC( - const struct compressionParameters* pThis, - const char* src, char* dst, - int srcSize, int dstSize) +static int +LZ4_compressBlockStreamHC(const struct compressionParameters* pThis, + const char* src, char* dst, + int srcSize, int dstSize) { return LZ4_compress_HC_continue(pThis->LZ4_streamHC, src, dst, srcSize, dstSize); } -static void LZ4_compressCleanupNoStream( - const struct compressionParameters* pThis) +static void +LZ4_compressCleanupNoStream(const struct compressionParameters* pThis) { (void)pThis; } -static void LZ4_compressCleanupStream( - const struct compressionParameters* pThis) +static void +LZ4_compressCleanupStream(const struct compressionParameters* pThis) { LZ4_freeStream(pThis->LZ4_stream); LZ4_freeStream(pThis->LZ4_dictStream); } -static void LZ4_compressCleanupStreamHC( - const struct compressionParameters* pThis) +static void +LZ4_compressCleanupStreamHC(const struct compressionParameters* pThis) { LZ4_freeStreamHC(pThis->LZ4_streamHC); LZ4_freeStreamHC(pThis->LZ4_dictStreamHC); } -static void LZ4_buildCompressionParameters( - struct compressionParameters* pParams, - int cLevel, const char* dictBuf, int dictSize) +static void +LZ4_buildCompressionParameters(struct compressionParameters* pParams, + int cLevel, + const char* dictBuf, int dictSize) { pParams->cLevel = cLevel; pParams->dictBuf = dictBuf; @@ -215,90 +304,32 @@ static void LZ4_buildCompressionParameters( } } -#define LZ4_isError(errcode) (errcode==0) - - -/* ************************************* -* Constants -***************************************/ -#ifndef LZ4_GIT_COMMIT_STRING -# define LZ4_GIT_COMMIT_STRING "" -#else -# define LZ4_GIT_COMMIT_STRING LZ4_EXPAND_AND_QUOTE(LZ4_GIT_COMMIT) -#endif - -#define NBSECONDS 3 -#define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */ -#define TIMELOOP_NANOSEC 1*1000000000ULL /* 1 second */ -#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */ -#define COOLPERIOD_SEC 10 -#define DECOMP_MULT 1 /* test decompression DECOMP_MULT times longer than compression */ - -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -#define LZ4_MAX_DICT_SIZE (64 KB) - -static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); - -static U32 g_compressibilityDefault = 50; - - -/* ************************************* -* console display -***************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ - -#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ - if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \ - { g_time = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stdout); } } -static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; -static clock_t g_time = 0; - - -/* ************************************* -* Exceptions -***************************************/ -#ifndef DEBUG -# define DEBUG 0 -#endif -#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); -#define EXM_THROW(error, ...) \ -{ \ - DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ - DISPLAYLEVEL(1, "Error %i : ", error); \ - DISPLAYLEVEL(1, __VA_ARGS__); \ - DISPLAYLEVEL(1, "\n"); \ - exit(error); \ -} - - -/* ************************************* -* Benchmark Parameters -***************************************/ -static U32 g_nbSeconds = NBSECONDS; -static size_t g_blockSize = 0; -int g_additionalParam = 0; -int g_benchSeparately = 0; -void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; } +typedef int (*DecFunction_f)(const char* src, char* dst, + int srcSize, int dstCapacity, + const char* dictStart, int dictSize); -void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; } +static LZ4F_dctx* g_dctx = NULL; -void BMK_setNbSeconds(unsigned nbSeconds) +static int +LZ4F_decompress_binding(const char* src, char* dst, + int srcSize, int dstCapacity, + const char* dictStart, int dictSize) { - g_nbSeconds = nbSeconds; - DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbSeconds); + size_t dstSize = (size_t)dstCapacity; + size_t readSize = (size_t)srcSize; + size_t const decStatus = LZ4F_decompress(g_dctx, + dst, &dstSize, + src, &readSize, + NULL /* dOptPtr */); + if ( (decStatus == 0) /* decompression successful */ + && ((int)readSize==srcSize) /* consume all input */ ) + return (int)dstSize; + /* else, error */ + return -1; + (void)dictStart; (void)dictSize; /* not compatible with dictionary yet */ } -void BMK_setBlockSize(size_t blockSize) { g_blockSize = blockSize; } - -void BMK_setBenchSeparately(int separate) { g_benchSeparately = (separate!=0); } - /* ******************************************************** * Bench functions @@ -321,24 +352,32 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, const size_t* fileSizes, U32 nbFiles, const char* dictBuf, int dictSize) { - size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; - U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; + size_t const blockSize = (g_blockSize>=32 && !g_decodeOnly ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; + U32 const maxNbBlocks = (U32)((srcSize + (blockSize-1)) / blockSize) + nbFiles; blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t)); - size_t const maxCompressedSize = LZ4_compressBound((int)srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ + size_t const maxCompressedSize = (size_t)LZ4_compressBound((int)srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ void* const compressedBuffer = malloc(maxCompressedSize); - void* const resultBuffer = malloc(srcSize); + size_t const decMultiplier = g_decodeOnly ? 255 : 1; + size_t const maxInSize = (size_t)LZ4_MAX_INPUT_SIZE / decMultiplier; + size_t const maxDecSize = srcSize < maxInSize ? srcSize * decMultiplier : LZ4_MAX_INPUT_SIZE; + void* const resultBuffer = malloc(maxDecSize); U32 nbBlocks; struct compressionParameters compP; /* checks */ if (!compressedBuffer || !resultBuffer || !blockTable) - EXM_THROW(31, "allocation error : not enough memory"); + END_PROCESS(31, "allocation error : not enough memory"); if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */ /* init */ LZ4_buildCompressionParameters(&compP, cLevel, dictBuf, dictSize); compP.initFunction(&compP); + if (g_dctx==NULL) { + LZ4F_createDecompressionContext(&g_dctx, LZ4F_VERSION); + if (g_dctx==NULL) + END_PROCESS(1, "allocation error - decompression state"); + } /* Init blockTable data */ { const char* srcPtr = (const char*)srcBuffer; @@ -351,6 +390,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, U32 const blockEnd = nbBlocks + nbBlocksforThisFile; for ( ; nbBlocks<blockEnd; nbBlocks++) { size_t const thisBlockSize = MIN(remaining, blockSize); + size_t const resMaxSize = thisBlockSize * decMultiplier; + size_t const resCapa = (thisBlockSize < maxInSize) ? resMaxSize : LZ4_MAX_INPUT_SIZE; blockTable[nbBlocks].srcPtr = srcPtr; blockTable[nbBlocks].cPtr = cPtr; blockTable[nbBlocks].resPtr = resPtr; @@ -358,29 +399,37 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, blockTable[nbBlocks].cRoom = (size_t)LZ4_compressBound((int)thisBlockSize); srcPtr += thisBlockSize; cPtr += blockTable[nbBlocks].cRoom; - resPtr += thisBlockSize; + resPtr += resCapa; remaining -= thisBlockSize; } } } /* warmimg up memory */ RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1); + /* decode-only mode : copy input to @compressedBuffer */ + if (g_decodeOnly) { + U32 blockNb; + for (blockNb=0; blockNb < nbBlocks; blockNb++) { + memcpy(blockTable[blockNb].cPtr, blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize); + blockTable[blockNb].cSize = blockTable[blockNb].srcSize; + } } + /* Bench */ { U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL); U64 const crcOrig = XXH64(srcBuffer, srcSize, 0); - UTIL_time_t coolTime; + UTIL_time_t coolTime = UTIL_getTime(); U64 const maxTime = (g_nbSeconds * TIMELOOP_NANOSEC) + 100; U32 nbCompressionLoops = (U32)((5 MB) / (srcSize+1)) + 1; /* conservative initial compression speed estimate */ U32 nbDecodeLoops = (U32)((200 MB) / (srcSize+1)) + 1; /* conservative initial decode speed estimate */ U64 totalCTime=0, totalDTime=0; - U32 cCompleted=0, dCompleted=0; + U32 cCompleted=(g_decodeOnly==1), dCompleted=0; # define NB_MARKS 4 const char* const marks[NB_MARKS] = { " |", " /", " =", "\\" }; U32 markNb = 0; - size_t cSize = 0; + size_t cSize = srcSize; + size_t totalRSize = srcSize; double ratio = 0.; - coolTime = UTIL_getTime(); DISPLAYLEVEL(2, "\r%79s\r", ""); while (!cCompleted || !dCompleted) { /* overheat protection */ @@ -391,8 +440,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, } /* Compression */ - DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize); - if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */ + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)totalRSize); + if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase compressed buffer */ UTIL_sleepMilli(1); /* give processor time to other processes */ UTIL_waitForNextTick(); @@ -408,7 +457,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, &compP, blockTable[blockNb].srcPtr, blockTable[blockNb].cPtr, (int)blockTable[blockNb].srcSize, (int)blockTable[blockNb].cRoom); - if (LZ4_isError(rSize)) EXM_THROW(1, "LZ4 compression failed"); + if (LZ4_isError(rSize)) END_PROCESS(1, "LZ4 compression failed"); blockTable[blockNb].cSize = rSize; } } { U64 const clockSpan = UTIL_clockSpanNano(clockStart); @@ -423,17 +472,18 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, } totalCTime += clockSpan; cCompleted = totalCTime>maxTime; - } } - - cSize = 0; - { U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; } - cSize += !cSize; /* avoid div by 0 */ - ratio = (double)srcSize / (double)cSize; - markNb = (markNb+1) % NB_MARKS; - DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r", - marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio, - ((double)srcSize / fastestC) * 1000 ); - + } + + cSize = 0; + { U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; } + cSize += !cSize; /* avoid div by 0 */ + ratio = (double)totalRSize / (double)cSize; + markNb = (markNb+1) % NB_MARKS; + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r", + marks[markNb], displayName, + (U32)totalRSize, (U32)cSize, ratio, + ((double)totalRSize / fastestC) * 1000 ); + } (void)fastestD; (void)crcOrig; /* unused when decompression disabled */ #if 1 /* Decompression */ @@ -443,17 +493,30 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, UTIL_waitForNextTick(); if (!dCompleted) { + const DecFunction_f decFunction = g_decodeOnly ? + LZ4F_decompress_binding : LZ4_decompress_safe_usingDict; + const char* const decString = g_decodeOnly ? + "LZ4F_decompress" : "LZ4_decompress_safe_usingDict"; UTIL_time_t const clockStart = UTIL_getTime(); U32 nbLoops; + for (nbLoops=0; nbLoops < nbDecodeLoops; nbLoops++) { U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) { - int const regenSize = LZ4_decompress_safe_usingDict( + size_t const inMaxSize = (size_t)INT_MAX / decMultiplier; + size_t const resCapa = (blockTable[blockNb].srcSize < inMaxSize) ? + blockTable[blockNb].srcSize * decMultiplier : + INT_MAX; + int const regenSize = decFunction( blockTable[blockNb].cPtr, blockTable[blockNb].resPtr, - (int)blockTable[blockNb].cSize, (int)blockTable[blockNb].srcSize, + (int)blockTable[blockNb].cSize, (int)resCapa, dictBuf, dictSize); if (regenSize < 0) { - DISPLAY("LZ4_decompress_safe_usingDict() failed on block %u \n", blockNb); + DISPLAY("%s() failed on block %u of size %u \n", + decString, blockNb, (unsigned)blockTable[blockNb].srcSize); + if (g_decodeOnly) + DISPLAY("Is input using LZ4 Frame format ? \n"); + END_PROCESS(2, "error during decoding"); break; } blockTable[blockNb].resSize = (size_t)regenSize; @@ -472,14 +535,22 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, dCompleted = totalDTime > (DECOMP_MULT*maxTime); } } + if (g_decodeOnly) { + unsigned u; + totalRSize = 0; + for (u=0; u<nbBlocks; u++) totalRSize += blockTable[u].resSize; + } markNb = (markNb+1) % NB_MARKS; + ratio = (double)totalRSize / (double)cSize; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", - marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio, - ((double)srcSize / fastestC) * 1000, - ((double)srcSize / fastestD) * 1000); - - /* CRC Checking */ - { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); + marks[markNb], displayName, + (U32)totalRSize, (U32)cSize, ratio, + ((double)totalRSize / fastestC) * 1000, + ((double)totalRSize / fastestD) * 1000); + + /* CRC Checking (not possible in decode-only mode)*/ + if (!g_decodeOnly) { + U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); if (crcOrig!=crcCheck) { size_t u; DISPLAY("\n!!! WARNING !!! %17s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck); @@ -594,21 +665,21 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize, continue; } f = fopen(fileNamesTable[n], "rb"); - if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]); + if (f==NULL) END_PROCESS(10, "impossible to open file %s", fileNamesTable[n]); DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]); if (fileSize > bufferSize-pos) { /* buffer too small - stop after this file */ fileSize = bufferSize-pos; nbFiles=n; } { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); - if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]); + if (readSize != (size_t)fileSize) END_PROCESS(11, "could not read %s", fileNamesTable[n]); pos += readSize; } fileSizes[n] = (size_t)fileSize; totalSize += (size_t)fileSize; fclose(f); } - if (totalSize == 0) EXM_THROW(12, "no data to bench"); + if (totalSize == 0) END_PROCESS(12, "no data to bench"); } static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, @@ -621,11 +692,11 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); char mfName[20] = {0}; - if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes"); + if (!fileSizes) END_PROCESS(12, "not enough memory for fileSizes"); /* Memory allocation & restrictions */ benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; - if (benchedSize==0) EXM_THROW(12, "not enough memory"); + if (benchedSize==0) END_PROCESS(12, "not enough memory"); if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad; if (benchedSize > LZ4_MAX_INPUT_SIZE) { benchedSize = LZ4_MAX_INPUT_SIZE; @@ -635,7 +706,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20)); } srcBuffer = malloc(benchedSize + !benchedSize); /* avoid alloc of zero */ - if (!srcBuffer) EXM_THROW(12, "not enough memory"); + if (!srcBuffer) END_PROCESS(12, "not enough memory"); /* Load input buffer */ BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles); @@ -663,7 +734,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility void* const srcBuffer = malloc(benchedSize); /* Memory allocation */ - if (!srcBuffer) EXM_THROW(21, "not enough memory"); + if (!srcBuffer) END_PROCESS(21, "not enough memory"); /* Fill input buffer */ RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0); @@ -677,7 +748,8 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility } -int BMK_benchFilesSeparately(const char** fileNamesTable, unsigned nbFiles, +static int +BMK_benchFilesSeparately(const char** fileNamesTable, unsigned nbFiles, int cLevel, int cLevelLast, const char* dictBuf, int dictSize) { @@ -685,7 +757,6 @@ int BMK_benchFilesSeparately(const char** fileNamesTable, unsigned nbFiles, if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX; if (cLevelLast > LZ4HC_CLEVEL_MAX) cLevelLast = LZ4HC_CLEVEL_MAX; if (cLevelLast < cLevel) cLevelLast = cLevel; - if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); for (fileNb=0; fileNb<nbFiles; fileNb++) BMK_benchFileTable(fileNamesTable+fileNb, 1, cLevel, cLevelLast, dictBuf, dictSize); @@ -700,45 +771,54 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, { double const compressibility = (double)g_compressibilityDefault / 100; char* dictBuf = NULL; - int dictSize = 0; + size_t dictSize = 0; if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX; + if (g_decodeOnly) { + DISPLAYLEVEL(2, "Benchmark Decompression (only) of LZ4 Frame \n"); + cLevelLast = cLevel; + } if (cLevelLast > LZ4HC_CLEVEL_MAX) cLevelLast = LZ4HC_CLEVEL_MAX; if (cLevelLast < cLevel) cLevelLast = cLevel; - if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); + if (cLevelLast > cLevel) + DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); if (dictFileName) { FILE* dictFile = NULL; - U64 dictFileSize = UTIL_getFileSize(dictFileName); - if (!dictFileSize) EXM_THROW(25, "Dictionary error : could not stat dictionary file"); + U64 const dictFileSize = UTIL_getFileSize(dictFileName); + if (!dictFileSize) + END_PROCESS(25, "Dictionary error : could not stat dictionary file"); + if (g_decodeOnly) + END_PROCESS(26, "Error : LZ4 Frame decoder mode not compatible with dictionary yet"); dictFile = fopen(dictFileName, "rb"); - if (!dictFile) EXM_THROW(25, "Dictionary error : could not open dictionary file"); + if (!dictFile) + END_PROCESS(25, "Dictionary error : could not open dictionary file"); if (dictFileSize > LZ4_MAX_DICT_SIZE) { dictSize = LZ4_MAX_DICT_SIZE; - if (UTIL_fseek(dictFile, dictFileSize - dictSize, SEEK_SET)) - EXM_THROW(25, "Dictionary error : could not seek dictionary file"); + if (UTIL_fseek(dictFile, (long)(dictFileSize - dictSize), SEEK_SET)) + END_PROCESS(25, "Dictionary error : could not seek dictionary file"); } else { - dictSize = (int)dictFileSize; + dictSize = (size_t)dictFileSize; } - dictBuf = (char *)malloc(dictSize); - if (!dictBuf) EXM_THROW(25, "Allocation error : not enough memory"); + dictBuf = (char*)malloc(dictSize); + if (!dictBuf) END_PROCESS(25, "Allocation error : not enough memory"); - if (fread(dictBuf, 1, dictSize, dictFile) != (size_t)dictSize) - EXM_THROW(25, "Dictionary error : could not read dictionary file"); + if (fread(dictBuf, 1, dictSize, dictFile) != dictSize) + END_PROCESS(25, "Dictionary error : could not read dictionary file"); fclose(dictFile); } if (nbFiles == 0) - BMK_syntheticTest(cLevel, cLevelLast, compressibility, dictBuf, dictSize); + BMK_syntheticTest(cLevel, cLevelLast, compressibility, dictBuf, (int)dictSize); else { if (g_benchSeparately) - BMK_benchFilesSeparately(fileNamesTable, nbFiles, cLevel, cLevelLast, dictBuf, dictSize); + BMK_benchFilesSeparately(fileNamesTable, nbFiles, cLevel, cLevelLast, dictBuf, (int)dictSize); else - BMK_benchFileTable(fileNamesTable, nbFiles, cLevel, cLevelLast, dictBuf, dictSize); + BMK_benchFileTable(fileNamesTable, nbFiles, cLevel, cLevelLast, dictBuf, (int)dictSize); } free(dictBuf); diff --git a/programs/bench.h b/programs/bench.h index 642d496..9b0f667 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -25,15 +25,29 @@ #include <stddef.h> +/* BMK_benchFiles() : + * Benchmark all files provided through array @fileNamesTable. + * All files must be valid, otherwise benchmark fails. + * Roundtrip measurements are done for each file individually, but + * unless BMK_setBenchSeparately() is set, all results are agglomerated. + * The method benchmarks all compression levels from @cLevelStart to @cLevelLast, + * both inclusive, providing one result per compression level. + * If @cLevelLast <= @cLevelStart, BMK_benchFiles() benchmarks @cLevelStart only. + * @dictFileName is optional, it's possible to provide NULL. + * When provided, compression and decompression use the specified file as dictionary. + * Only one dictionary can be provided, in which case it's applied to all benchmarked files. +**/ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - int cLevel, int cLevelLast, + int cLevelStart, int cLevelLast, const char* dictFileName); /* Set Parameters */ -void BMK_setNbSeconds(unsigned nbLoops); -void BMK_setBlockSize(size_t blockSize); -void BMK_setAdditionalParam(int additionalParam); -void BMK_setNotificationLevel(unsigned level); -void BMK_setBenchSeparately(int separate); +void BMK_setNbSeconds(unsigned nbSeconds); /* minimum benchmark duration, in seconds, for both compression and decompression */ +void BMK_setBlockSize(size_t blockSize); /* Internally cut input file(s) into independent blocks of specified size */ +void BMK_setNotificationLevel(unsigned level); /* Influence verbosity level */ +void BMK_setBenchSeparately(int separate); /* When providing multiple files, output one result per file */ +void BMK_setDecodeOnlyMode(int set); /* v1.9.4+: set benchmark mode to decode only */ + +void BMK_setAdditionalParam(int additionalParam); /* hidden param, influence output format, for python parsing */ #endif /* BENCH_H_125623623633 */ diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 254a6ce..42132b9 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -377,8 +377,12 @@ int main(int argc, const char** argv) if (argument[1]=='-') { if (!strcmp(argument, "--")) { all_arguments_are_files = 1; continue; } if (!strcmp(argument, "--compress")) { mode = om_compress; continue; } - if ((!strcmp(argument, "--decompress")) - || (!strcmp(argument, "--uncompress"))) { mode = om_decompress; continue; } + if ( (!strcmp(argument, "--decompress")) + || (!strcmp(argument, "--uncompress"))) { + if (mode != om_bench) mode = om_decompress; + BMK_setDecodeOnlyMode(1); + continue; + } if (!strcmp(argument, "--multiple")) { multiple_inputs = 1; continue; } if (!strcmp(argument, "--test")) { mode = om_test; continue; } if (!strcmp(argument, "--force")) { LZ4IO_setOverwrite(prefs, 1); continue; } @@ -478,7 +482,10 @@ int main(int argc, const char** argv) case 'l': legacy_format = 1; blockSize = 8 MB; break; /* Decoding */ - case 'd': mode = om_decompress; break; + case 'd': + if (mode != om_bench) mode = om_decompress; + BMK_setDecodeOnlyMode(1); + break; /* Force stdout, even if stdout==console */ case 'c': diff --git a/tests/Makefile b/tests/Makefile index 469db9a..d804f25 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -476,6 +476,9 @@ test-lz4-testmode: FPREFIX = tmp-ltm test-lz4-testmode: lz4 datagen @echo "\n ---- bench mode ----" $(LZ4) -bi0 + $(DATAGEN) > $(FPREFIX) + $(LZ4) -f $(FPREFIX) -c > $(FPREFIX).lz4 + $(LZ4) -bdi0 $(FPREFIX).lz4 # test benchmark decode-only mode @echo "\n ---- test mode ----" ! $(DATAGEN) | $(LZ4) -t ! $(DATAGEN) | $(LZ4) -tf diff --git a/tests/fullbench.c b/tests/fullbench.c index ec20dcb..9c13996 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -397,7 +397,8 @@ static int local_LZ4F_decompress_followHint(const char* src, char* dst, int srcS size_t outRemaining = maxOutSize - outPos; for (;;) { - size_t const sizeHint = LZ4F_decompress(g_dCtx, dst+outPos, &outRemaining, src+inPos, &inSize, NULL); + size_t const sizeHint = + LZ4F_decompress(g_dCtx, dst+outPos, &outRemaining, src+inPos, &inSize, NULL); assert(!LZ4F_isError(sizeHint)); inPos += inSize; |