From 379c1a10cad71b004a4fa95a482c47ca0fa18835 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 30 Jan 2022 23:01:50 -0800 Subject: Introduce MIN and MAX bounds to LZ4_MEMORY_USAGE ensure that `frametest` works fine with these values, notably with low LZ4_MEMORY_USAGE (dict test notably) following suggestions from @t-mat at #1016 --- lib/lz4.c | 2 +- lib/lz4.h | 16 +++++++++++++--- lib/lz4frame.c | 4 ++-- tests/frametest.c | 19 ++++++++++--------- 4 files changed, 26 insertions(+), 15 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index 95bd349..a2272cf 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -867,7 +867,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated( const char* const source, char* const dest, const int inputSize, - int *inputConsumed, /* only written when outputDirective == fillOutput */ + int* inputConsumed, /* only written when outputDirective == fillOutput */ const int maxOutputSize, const limitedOutput_directive outputDirective, const tableType_t tableType, diff --git a/lib/lz4.h b/lib/lz4.h index a520adc..50f1d7a 100644 --- a/lib/lz4.h +++ b/lib/lz4.h @@ -118,15 +118,25 @@ LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; **************************************/ /*! * LZ4_MEMORY_USAGE : - * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) - * Increasing memory usage improves compression ratio. - * Reduced memory usage may improve speed, thanks to better cache locality. + * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; ) + * Increasing memory usage improves compression ratio, at the cost of speed. + * Reduced memory usage may improve speed at the cost of ratio, thanks to better cache locality. * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ #ifndef LZ4_MEMORY_USAGE # define LZ4_MEMORY_USAGE 14 #endif +#define LZ4_MEMORY_USAGE_MIN 10 +#define LZ4_MEMORY_USAGE_MAX 20 + +#if (LZ4_MEMORY_USAGE < LZ4_MEMORY_USAGE_MIN) +# error "LZ4_MEMORY_USAGE is too small !" +#endif + +#if (LZ4_MEMORY_USAGE > LZ4_MEMORY_USAGE_MAX) +# error "LZ4_MEMORY_USAGE is too large !" +#endif /*-************************************ * Simple Functions diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 73f21fc..f4ea02a 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -485,12 +485,12 @@ struct LZ4F_CDict_s { * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. * LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. * LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. - * `dictBuffer` can be released after LZ4F_CDict creation, since its content is copied within CDict + * @dictBuffer can be released after LZ4F_CDict creation, since its content is copied within CDict * @return : digested dictionary for compression, or NULL if failed */ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize) { const char* dictStart = (const char*)dictBuffer; - LZ4F_CDict* cdict = (LZ4F_CDict*) ALLOC(sizeof(*cdict)); + LZ4F_CDict* const cdict = (LZ4F_CDict*) ALLOC(sizeof(*cdict)); DEBUGLOG(4, "LZ4F_createCDict"); if (!cdict) return NULL; if (dictSize > 64 KB) { diff --git a/tests/frametest.c b/tests/frametest.c index e0fff0e..09def51 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -535,8 +535,9 @@ int basicTests(U32 seed, double compressibility) } /* Dictionary compression test */ - { size_t const dictSize = 63 KB; - size_t const dstCapacity = LZ4F_compressFrameBound(dictSize, NULL); + { size_t const dictSize = 7 KB; /* small enough for LZ4_MEMORY_USAGE == 10 */ + size_t const srcSize = 65 KB; /* must be > 64 KB to avoid short-size optimizations */ + size_t const dstCapacity = LZ4F_compressFrameBound(srcSize, NULL); size_t cSizeNoDict, cSizeWithDict; LZ4F_CDict* const cdict = LZ4F_createCDict(CNBuffer, dictSize); if (cdict == NULL) goto _output_error; @@ -545,7 +546,7 @@ int basicTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with NULL dict : "); CHECK_V(cSizeNoDict, LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity, - CNBuffer, dictSize, + CNBuffer, srcSize, NULL, NULL) ); DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeNoDict); @@ -554,19 +555,19 @@ int basicTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict : "); CHECK_V(cSizeWithDict, LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity, - CNBuffer, dictSize, + CNBuffer, srcSize, cdict, NULL) ); DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n", - (unsigned)dictSize, (unsigned)cSizeWithDict); - if ((LZ4_DISTANCE_MAX > dictSize) && (cSizeWithDict >= cSizeNoDict)) { + (unsigned)srcSize, (unsigned)cSizeWithDict); + if (cSizeWithDict > cSizeNoDict) { DISPLAYLEVEL(3, "cSizeWithDict (%zu) should have been more compact than cSizeNoDict(%zu) \n", cSizeWithDict, cSizeNoDict); goto _output_error; /* must be more efficient */ } - crcOrig = XXH64(CNBuffer, dictSize, 0); + crcOrig = XXH64(CNBuffer, srcSize, 0); DISPLAYLEVEL(3, "LZ4F_decompress_usingDict : "); { LZ4F_dctx* dctx; - size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH; + size_t decodedSize = srcSize; size_t compressedSize = cSizeWithDict; CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) ); CHECK( LZ4F_decompress_usingDict(dctx, @@ -575,7 +576,7 @@ int basicTests(U32 seed, double compressibility) CNBuffer, dictSize, NULL) ); if (compressedSize != cSizeWithDict) goto _output_error; - if (decodedSize != dictSize) goto _output_error; + if (decodedSize != srcSize) goto _output_error; { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0); if (crcDest != crcOrig) goto _output_error; } DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize); -- cgit v0.12