diff options
author | Yann Collet <Cyan4973@users.noreply.github.com> | 2022-01-31 23:57:46 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-01-31 23:57:46 (GMT) |
commit | 7601d57f878649eea61fa2629424e0059d1591c6 (patch) | |
tree | 14b092852f4877522b6fdd058d56ef418b5e59fa | |
parent | 756693083a0ac4c176101ca021809fe15d3a69ee (diff) | |
parent | 4f984e45a5a36c446d959e1c790d90cef5d5dcb8 (diff) | |
download | lz4-7601d57f878649eea61fa2629424e0059d1591c6.zip lz4-7601d57f878649eea61fa2629424e0059d1591c6.tar.gz lz4-7601d57f878649eea61fa2629424e0059d1591c6.tar.bz2 |
Merge pull request #1061 from lz4/memory_usage
Introduce MIN and MAX bounds to LZ4_MEMORY_USAGE
-rw-r--r-- | .github/workflows/ci.yml | 10 | ||||
-rw-r--r-- | lib/lz4.c | 2 | ||||
-rw-r--r-- | lib/lz4.h | 19 | ||||
-rw-r--r-- | lib/lz4frame.c | 4 | ||||
-rw-r--r-- | lib/lz4hc.c | 2 | ||||
-rw-r--r-- | tests/Makefile | 8 | ||||
-rw-r--r-- | tests/frametest.c | 19 |
7 files changed, 46 insertions, 18 deletions
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5c0afb1..fe9f479 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -236,6 +236,13 @@ jobs: - name: LZ4 frame test (32-bit) run: make V=1 -C tests test-frametest32 + lz4-memory-usage: + name: test different values of LZ4_MEMORY_USAGE + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 # https://github.com/actions/checkout + - name: LZ4_MEMORY_USAGE + run: make V=1 -C tests test-compile-with-lz4-memory-usage # Custom LZ4_DISTANCE_MAX ; lz4-wlib (CLI linked to dynamic library); LZ4_USER_MEMORY_FUNCTIONS lz4-custom-distance: @@ -244,7 +251,7 @@ jobs: steps: - uses: actions/checkout@v2 # https://github.com/actions/checkout - - name: custom LZ4_DISTANCE_MAX + - name: custom LZ4_DISTANCE_MAX; test LZ4_USER_MEMORY_FUNCTIONS run: | MOREFLAGS='-DLZ4_DISTANCE_MAX=8000' make V=1 check make V=1 clean @@ -392,7 +399,6 @@ jobs: - name: unicode lint run: bash ./tests/unicode_lint.sh - lz4-examples: name: make examples runs-on: ubuntu-latest @@ -867,7 +867,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated( const char* const source, char* const dest, const int inputSize, - int *inputConsumed, /* only written when outputDirective == fillOutput */ + int* inputConsumed, /* only written when outputDirective == fillOutput */ const int maxOutputSize, const limitedOutput_directive outputDirective, const tableType_t tableType, @@ -116,17 +116,28 @@ LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; /*-************************************ * Tuning parameter **************************************/ +#define LZ4_MEMORY_USAGE_MIN 10 +#define LZ4_MEMORY_USAGE_DEFAULT 14 +#define LZ4_MEMORY_USAGE_MAX 20 + /*! * LZ4_MEMORY_USAGE : - * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) - * Increasing memory usage improves compression ratio. - * Reduced memory usage may improve speed, thanks to better cache locality. + * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; ) + * Increasing memory usage improves compression ratio, at the cost of speed. + * Reduced memory usage may improve speed at the cost of ratio, thanks to better cache locality. * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ #ifndef LZ4_MEMORY_USAGE -# define LZ4_MEMORY_USAGE 14 +# define LZ4_MEMORY_USAGE LZ4_MEMORY_USAGE_DEFAULT +#endif + +#if (LZ4_MEMORY_USAGE < LZ4_MEMORY_USAGE_MIN) +# error "LZ4_MEMORY_USAGE is too small !" #endif +#if (LZ4_MEMORY_USAGE > LZ4_MEMORY_USAGE_MAX) +# error "LZ4_MEMORY_USAGE is too large !" +#endif /*-************************************ * Simple Functions diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 73f21fc..f4ea02a 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -485,12 +485,12 @@ struct LZ4F_CDict_s { * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. * LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. * LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. - * `dictBuffer` can be released after LZ4F_CDict creation, since its content is copied within CDict + * @dictBuffer can be released after LZ4F_CDict creation, since its content is copied within CDict * @return : digested dictionary for compression, or NULL if failed */ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize) { const char* dictStart = (const char*)dictBuffer; - LZ4F_CDict* cdict = (LZ4F_CDict*) ALLOC(sizeof(*cdict)); + LZ4F_CDict* const cdict = (LZ4F_CDict*) ALLOC(sizeof(*cdict)); DEBUGLOG(4, "LZ4F_createCDict"); if (!cdict) return NULL; if (dictSize > 64 KB) { diff --git a/lib/lz4hc.c b/lib/lz4hc.c index ee6fc41..99650a6 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -277,6 +277,8 @@ LZ4HC_InsertAndGetWiderMatch ( /* do nothing */ } else if (matchIndex >= dictLimit) { /* within current Prefix */ const BYTE* const matchPtr = base + matchIndex; + DEBUGLOG(2, "matchPtr = %p", matchPtr); + DEBUGLOG(2, "lowPrefixPtr = %p", lowPrefixPtr); assert(matchPtr >= lowPrefixPtr); assert(matchPtr < ip); assert(longest >= 1); diff --git a/tests/Makefile b/tests/Makefile index 5fc6fc6..b4d40ca 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -174,14 +174,22 @@ test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-install test32: CFLAGS+=-m32 test32: test +.PHONY: test-amalgamation test-amalgamation: lz4_all.o lz4_all.c: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(CAT) $^ > $@ +.PHONY: test-install test-install: lz4 lib liblz4.pc lz4_root=.. ./test_install.sh +.PHONY: test-compile-with-lz4-memory-usage +test-compile-with-lz4-memory-usage: + $(MAKE) clean; CFLAGS=-O0 CPPFLAGS=-D'LZ4_MEMORY_USAGE=LZ4_MEMORY_USAGE_MIN' $(MAKE) all + $(MAKE) clean; CFLAGS=-O0 CPPFLAGS=-D'LZ4_MEMORY_USAGE=LZ4_MEMORY_USAGE_MAX' $(MAKE) all + +.PHONY: test-lz4-sparse test-lz4-sparse: lz4 datagen @echo "\n ---- test sparse file support ----" $(DATAGEN) -g5M -P100 > tmplsdg5M diff --git a/tests/frametest.c b/tests/frametest.c index e0fff0e..09def51 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -535,8 +535,9 @@ int basicTests(U32 seed, double compressibility) } /* Dictionary compression test */ - { size_t const dictSize = 63 KB; - size_t const dstCapacity = LZ4F_compressFrameBound(dictSize, NULL); + { size_t const dictSize = 7 KB; /* small enough for LZ4_MEMORY_USAGE == 10 */ + size_t const srcSize = 65 KB; /* must be > 64 KB to avoid short-size optimizations */ + size_t const dstCapacity = LZ4F_compressFrameBound(srcSize, NULL); size_t cSizeNoDict, cSizeWithDict; LZ4F_CDict* const cdict = LZ4F_createCDict(CNBuffer, dictSize); if (cdict == NULL) goto _output_error; @@ -545,7 +546,7 @@ int basicTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with NULL dict : "); CHECK_V(cSizeNoDict, LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity, - CNBuffer, dictSize, + CNBuffer, srcSize, NULL, NULL) ); DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeNoDict); @@ -554,19 +555,19 @@ int basicTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict : "); CHECK_V(cSizeWithDict, LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity, - CNBuffer, dictSize, + CNBuffer, srcSize, cdict, NULL) ); DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n", - (unsigned)dictSize, (unsigned)cSizeWithDict); - if ((LZ4_DISTANCE_MAX > dictSize) && (cSizeWithDict >= cSizeNoDict)) { + (unsigned)srcSize, (unsigned)cSizeWithDict); + if (cSizeWithDict > cSizeNoDict) { DISPLAYLEVEL(3, "cSizeWithDict (%zu) should have been more compact than cSizeNoDict(%zu) \n", cSizeWithDict, cSizeNoDict); goto _output_error; /* must be more efficient */ } - crcOrig = XXH64(CNBuffer, dictSize, 0); + crcOrig = XXH64(CNBuffer, srcSize, 0); DISPLAYLEVEL(3, "LZ4F_decompress_usingDict : "); { LZ4F_dctx* dctx; - size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH; + size_t decodedSize = srcSize; size_t compressedSize = cSizeWithDict; CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) ); CHECK( LZ4F_decompress_usingDict(dctx, @@ -575,7 +576,7 @@ int basicTests(U32 seed, double compressibility) CNBuffer, dictSize, NULL) ); if (compressedSize != cSizeWithDict) goto _output_error; - if (decodedSize != dictSize) goto _output_error; + if (decodedSize != srcSize) goto _output_error; { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0); if (crcDest != crcOrig) goto _output_error; } DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize); |