Merge pull request #1061 from lz4/memory_usage

Introduce MIN and MAX bounds to LZ4_MEMORY_USAGE

7 files changed, 46 insertions, 18 deletions

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5c0afb1..fe9f479 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -236,6 +236,13 @@ jobs:
     - name: LZ4 frame test (32-bit)
       run: make V=1 -C tests test-frametest32
 
+  lz4-memory-usage:
+    name: test different values of LZ4_MEMORY_USAGE
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2 # https://github.com/actions/checkout
+    - name: LZ4_MEMORY_USAGE
+      run: make V=1 -C tests test-compile-with-lz4-memory-usage
 
   # Custom LZ4_DISTANCE_MAX ; lz4-wlib (CLI linked to dynamic library); LZ4_USER_MEMORY_FUNCTIONS
   lz4-custom-distance:
@@ -244,7 +251,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2 # https://github.com/actions/checkout
 
-    - name: custom LZ4_DISTANCE_MAX
+    - name: custom LZ4_DISTANCE_MAX; test LZ4_USER_MEMORY_FUNCTIONS
       run: |
         MOREFLAGS='-DLZ4_DISTANCE_MAX=8000' make V=1 check
         make V=1 clean
@@ -392,7 +399,6 @@ jobs:
     - name: unicode lint
       run: bash ./tests/unicode_lint.sh
 
-
   lz4-examples:
     name: make examples
     runs-on: ubuntu-latest
diff --git a/lib/lz4.c b/lib/lz4.c
index 95bd349..a2272cf 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -867,7 +867,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated(
                  const char* const source,
                  char* const dest,
                  const int inputSize,
-                 int *inputConsumed, /* only written when outputDirective == fillOutput */
+                 int*  inputConsumed, /* only written when outputDirective == fillOutput */
                  const int maxOutputSize,
                  const limitedOutput_directive outputDirective,
                  const tableType_t tableType,
diff --git a/lib/lz4.h b/lib/lz4.h
index a520adc..7c401f6 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -116,17 +116,28 @@ LZ4LIB_API const char* LZ4_versionString (void);   /**< library version string;
 /*-************************************
 *  Tuning parameter
 **************************************/
+#define LZ4_MEMORY_USAGE_MIN 10
+#define LZ4_MEMORY_USAGE_DEFAULT 14
+#define LZ4_MEMORY_USAGE_MAX 20
+
 /*!
  * LZ4_MEMORY_USAGE :
- * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
- * Increasing memory usage improves compression ratio.
- * Reduced memory usage may improve speed, thanks to better cache locality.
+ * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; )
+ * Increasing memory usage improves compression ratio, at the cost of speed.
+ * Reduced memory usage may improve speed at the cost of ratio, thanks to better cache locality.
  * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
  */
 #ifndef LZ4_MEMORY_USAGE
-# define LZ4_MEMORY_USAGE 14
+# define LZ4_MEMORY_USAGE LZ4_MEMORY_USAGE_DEFAULT
+#endif
+
+#if (LZ4_MEMORY_USAGE < LZ4_MEMORY_USAGE_MIN)
+#  error "LZ4_MEMORY_USAGE is too small !"
 #endif
 
+#if (LZ4_MEMORY_USAGE > LZ4_MEMORY_USAGE_MAX)
+#  error "LZ4_MEMORY_USAGE is too large !"
+#endif
 
 /*-************************************
 *  Simple Functions
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 73f21fc..f4ea02a 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -485,12 +485,12 @@ struct LZ4F_CDict_s {
  *  When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
  *  LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
  *  LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
- * `dictBuffer` can be released after LZ4F_CDict creation, since its content is copied within CDict
+ * @dictBuffer can be released after LZ4F_CDict creation, since its content is copied within CDict
  * @return : digested dictionary for compression, or NULL if failed */
 LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
 {
     const char* dictStart = (const char*)dictBuffer;
-    LZ4F_CDict* cdict = (LZ4F_CDict*) ALLOC(sizeof(*cdict));
+    LZ4F_CDict* const cdict = (LZ4F_CDict*) ALLOC(sizeof(*cdict));
     DEBUGLOG(4, "LZ4F_createCDict");
     if (!cdict) return NULL;
     if (dictSize > 64 KB) {
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index ee6fc41..99650a6 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -277,6 +277,8 @@ LZ4HC_InsertAndGetWiderMatch (
             /* do nothing */
         } else if (matchIndex >= dictLimit) {   /* within current Prefix */
             const BYTE* const matchPtr = base + matchIndex;
+            DEBUGLOG(2, "matchPtr = %p", matchPtr);
+            DEBUGLOG(2, "lowPrefixPtr = %p", lowPrefixPtr);
             assert(matchPtr >= lowPrefixPtr);
             assert(matchPtr < ip);
             assert(longest >= 1);
diff --git a/tests/Makefile b/tests/Makefile
index 5fc6fc6..b4d40ca 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -174,14 +174,22 @@ test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-install
 test32: CFLAGS+=-m32
 test32: test
 
+.PHONY: test-amalgamation
 test-amalgamation: lz4_all.o
 
 lz4_all.c: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c
 	$(CAT) $^ > $@
 
+.PHONY: test-install
 test-install: lz4 lib liblz4.pc
 	lz4_root=.. ./test_install.sh
 
+.PHONY: test-compile-with-lz4-memory-usage
+test-compile-with-lz4-memory-usage:
+	$(MAKE) clean; CFLAGS=-O0 CPPFLAGS=-D'LZ4_MEMORY_USAGE=LZ4_MEMORY_USAGE_MIN' $(MAKE) all
+	$(MAKE) clean; CFLAGS=-O0 CPPFLAGS=-D'LZ4_MEMORY_USAGE=LZ4_MEMORY_USAGE_MAX' $(MAKE) all
+
+.PHONY: test-lz4-sparse
 test-lz4-sparse: lz4 datagen
 	@echo "\n ---- test sparse file support ----"
 	$(DATAGEN) -g5M  -P100 > tmplsdg5M
diff --git a/tests/frametest.c b/tests/frametest.c
index e0fff0e..09def51 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -535,8 +535,9 @@ int basicTests(U32 seed, double compressibility)
     }
 
     /* Dictionary compression test */
-    {   size_t const dictSize = 63 KB;
-        size_t const dstCapacity = LZ4F_compressFrameBound(dictSize, NULL);
+    {   size_t const dictSize = 7 KB; /* small enough for LZ4_MEMORY_USAGE == 10 */
+        size_t const srcSize = 65 KB; /* must be > 64 KB to avoid short-size optimizations */
+        size_t const dstCapacity = LZ4F_compressFrameBound(srcSize, NULL);
         size_t cSizeNoDict, cSizeWithDict;
         LZ4F_CDict* const cdict = LZ4F_createCDict(CNBuffer, dictSize);
         if (cdict == NULL) goto _output_error;
@@ -545,7 +546,7 @@ int basicTests(U32 seed, double compressibility)
         DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with NULL dict : ");
         CHECK_V(cSizeNoDict,
                 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
-                                              CNBuffer, dictSize,
+                                              CNBuffer, srcSize,
                                               NULL, NULL) );
         DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeNoDict);
 
@@ -554,19 +555,19 @@ int basicTests(U32 seed, double compressibility)
         DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict : ");
         CHECK_V(cSizeWithDict,
                 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
-                                              CNBuffer, dictSize,
+                                              CNBuffer, srcSize,
                                               cdict, NULL) );
         DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
-                        (unsigned)dictSize, (unsigned)cSizeWithDict);
-        if ((LZ4_DISTANCE_MAX > dictSize) && (cSizeWithDict >= cSizeNoDict)) {
+                        (unsigned)srcSize, (unsigned)cSizeWithDict);
+        if (cSizeWithDict > cSizeNoDict) {
             DISPLAYLEVEL(3, "cSizeWithDict (%zu) should have been more compact than cSizeNoDict(%zu) \n", cSizeWithDict, cSizeNoDict);
             goto _output_error;  /* must be more efficient */
         }
-        crcOrig = XXH64(CNBuffer, dictSize, 0);
+        crcOrig = XXH64(CNBuffer, srcSize, 0);
 
         DISPLAYLEVEL(3, "LZ4F_decompress_usingDict : ");
         {   LZ4F_dctx* dctx;
-            size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
+            size_t decodedSize = srcSize;
             size_t compressedSize = cSizeWithDict;
             CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
             CHECK( LZ4F_decompress_usingDict(dctx,
@@ -575,7 +576,7 @@ int basicTests(U32 seed, double compressibility)
                                         CNBuffer, dictSize,
                                         NULL) );
             if (compressedSize != cSizeWithDict) goto _output_error;
-            if (decodedSize != dictSize) goto _output_error;
+            if (decodedSize != srcSize) goto _output_error;
             { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0);
               if (crcDest != crcOrig) goto _output_error; }
             DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);