Several changes to address a few concerns from Yann. See Google Group LZ4c topic 'Reusing compression/decompression resources' for details.

1 files changed, 73 insertions, 21 deletions

diff --git a/examples/compress_functions.c b/examples/compress_functions.c
index 08913ef..d0dca13 100644
--- a/examples/compress_functions.c
+++ b/examples/compress_functions.c
@@ -5,7 +5,7 @@
  * Description: A program to demonstrate the various compression functions involved in when using LZ4_compress_default().  The idea
  *              is to show how each step in the call stack can be used directly, if desired.  There is also some benchmarking for
  *              each function to demonstrate the (probably lack of) performance difference when jumping the stack.
- *              (If you're new to lz4, please read basics.c to understand the fundamentals)
+ *              (If you're new to lz4, please read simple_buffer.c to understand the fundamentals)
  *
  *              The call stack (before theoretical compiler optimizations) for LZ4_compress_default is as follows:
  *                LZ4_compress_default
@@ -28,6 +28,27 @@
  *                directly can help avoid some test cases and branching which might be useful in some implementation-specific
  *                situations, but you really need to know what you're doing AND what you're asking lz4 to do!  You also need a
  *                wrapper function because this function isn't exposed with lz4.h.
+ *
+ *              The call stack for decompression functions is shallow.  There are 2 options:
+ *                LZ4_decompress_safe  ||  LZ4_decompress_fast
+ *                  LZ4_decompress_generic
+ *
+ *               LZ4_decompress_safe
+ *                 This is the recommended function for decompressing data.  It is considered safe because the caller specifies
+ *                 both the size of the compresssed buffer to read as well as the maximum size of the output (decompressed) buffer
+ *                 instead of just the latter.
+ *               LZ4_decompress_fast
+ *                 Again, despite its name it's not a "fast" version of decompression.  It simply frees the caller of sending the
+ *                 size of the compressed buffer (it will simply be read-to-end, hence it's non-safety).
+ *               LZ4_decompress_generic
+ *                 This is the generic function that both of the LZ4_decompress_* functions above end up calling.  Calling this
+ *                 directly is not advised, period.  Furthermore, it is a static inline function in lz4.c, so there isn't a symbol
+ *                 exposed for anyone using lz4.h to utilize.
+ *
+ *               Special Note About Decompression:
+ *               Using the LZ4_decompress_safe() function protects against malicious (user) input.  If you are using data from a
+ *               trusted source, or if your program is the producer (P) as well as its consumer (C) in a PC or MPMC setup, you can
+ *               safely use the LZ4_decompress_fast function
  */
 
 /* Since lz4 compiles with c99 and not gnu/std99 we need to enable POSIX linking for time.h structs and functions. */
@@ -56,7 +77,8 @@
 #define ID__LZ4_COMPRESS_FAST           2
 #define ID__LZ4_COMPRESS_FAST_EXTSTATE  3
 #define ID__LZ4_COMPRESS_GENERIC        4
-#define ID__LZ4_DECOMPRESS_FAST         5
+#define ID__LZ4_DECOMPRESS_SAFE         5
+#define ID__LZ4_DECOMPRESS_FAST         6
 
 
 
@@ -84,7 +106,16 @@ void usage(const char *message) {
 /*
  * Runs the benchmark for LZ4_compress_* based on function_id.
  */
-uint64_t bench(const char *known_good_dst, const int function_id, const int iterations, const char *src, char *dst, const int src_size, const int max_dst_size) {
+uint64_t bench(
+    const char *known_good_dst,
+    const int function_id,
+    const int iterations,
+    const char *src,
+    char *dst,
+    const size_t src_size,
+    const size_t max_dst_size,
+    const size_t comp_size
+  ) {
   uint64_t time_taken = 0;
   int rv = 0;
   const int warm_up = 5000;
@@ -154,12 +185,25 @@ uint64_t bench(const char *known_good_dst, const int function_id, const int iter
 //      }
 //      break;
 
+    case ID__LZ4_DECOMPRESS_SAFE:
+      printf("Starting benchmark for function: LZ4_decompress_safe()\n");
+      for(int junk=0; junk<warm_up; junk++)
+        rv = LZ4_decompress_safe(src, dst, comp_size, src_size);
+      if (rv < 1)
+        run_screaming("Couldn't run LZ4_decompress_safe()... error code received is in exit code.", rv);
+      if (memcmp(known_good_dst, dst, src_size) != 0)
+        run_screaming("According to memcmp(), the compressed dst we got doesn't match the known_good_dst... ruh roh.", 1);
+      clock_gettime(CLOCK_MONOTONIC, &start);
+      for (int i=1; i<=iterations; i++)
+        LZ4_decompress_safe(src, dst, comp_size, src_size);
+      break;
+
     case ID__LZ4_DECOMPRESS_FAST:
       printf("Starting benchmark for function: LZ4_decompress_fast()\n");
       for(int junk=0; junk<warm_up; junk++)
         rv = LZ4_decompress_fast(src, dst, src_size);
       if (rv < 1)
-        run_screaming("Couldn't run LZ4_compress_generic()... error code received is in exit code.", rv);
+        run_screaming("Couldn't run LZ4_decompress_fast()... error code received is in exit code.", rv);
       if (memcmp(known_good_dst, dst, src_size) != 0)
         run_screaming("According to memcmp(), the compressed dst we got doesn't match the known_good_dst... ruh roh.", 1);
       clock_gettime(CLOCK_MONOTONIC, &start);
@@ -200,9 +244,9 @@ int main(int argc, char **argv) {
   // First we will create 2 sources (char *) of 2000 bytes each.  One normal text, the other highly-compressible text.
   const char *src    = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed luctus purus et risus vulputate, et mollis orci ullamcorper. Nulla facilisi. Fusce in ligula sed purus varius aliquet interdum vitae justo. Proin quis diam velit. Nulla varius iaculis auctor. Cras volutpat, justo eu dictum pulvinar, elit sem porttitor metus, et imperdiet metus sapien et ante. Nullam nisi nulla, ornare eu tristique eu, dignissim vitae diam. Nulla sagittis porta libero, a accumsan felis sagittis scelerisque.  Integer laoreet eleifend congue. Etiam rhoncus leo vel dolor fermentum, quis luctus nisl iaculis. Praesent a erat sapien. Aliquam semper mi in lorem ultrices ultricies. Lorem ipsum dolor sit amet, consectetur adipiscing elit. In feugiat risus sed enim ultrices, at sodales nulla tristique. Maecenas eget pellentesque justo, sed pellentesque lectus. Fusce sagittis sit amet elit vel varius. Donec sed ligula nec ligula vulputate rutrum sed ut lectus. Etiam congue pharetra leo vitae cursus. Morbi enim ante, porttitor ut varius vel, tincidunt quis justo. Nunc iaculis, risus id ultrices semper, metus est efficitur ligula, vel posuere risus nunc eget purus. Ut lorem turpis, condimentum at sem sed, porta aliquam turpis. In ut sapien a nulla dictum tincidunt quis sit amet lorem. Fusce at est egestas, luctus neque eu, consectetur tortor. Phasellus eleifend ultricies nulla ac lobortis.  Morbi maximus quam cursus vehicula iaculis. Maecenas cursus vel justo ut rutrum. Curabitur magna orci, dignissim eget dapibus vitae, finibus id lacus. Praesent rhoncus mattis augue vitae bibendum. Praesent porta mauris non ultrices fermentum. Quisque vulputate ipsum in sodales pulvinar. Aliquam nec mollis felis. Donec vitae augue pulvinar, congue nisl sed, pretium purus. Fusce lobortis mi ac neque scelerisque semper. Pellentesque vel est vitae magna aliquet aliquet. Nam non dolor. Nulla facilisi. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Morbi ac lacinia felis metus.";
   const char *hc_src = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
-  // Set and derive sizes.
-  const int src_size = 2000;
-  const int max_dst_size = LZ4_compressBound(src_size);
+  // Set and derive sizes.  Since we're using strings, use strlen() + 1 for \0.
+  const size_t src_size = strlen(src) + 1;
+  const size_t max_dst_size = LZ4_compressBound(src_size);
   int bytes_returned = 0;
   // Now build allocations for the data we'll be playing with.
   char *dst               = calloc(1, max_dst_size);
@@ -212,10 +256,14 @@ int main(int argc, char **argv) {
     run_screaming("Couldn't allocate memory for the destination buffers.  Sad :(", 1);
 
   // Create known-good buffers to verify our tests with other functions will produce the same results.
-  if (LZ4_compress_default(src, known_good_dst, src_size, max_dst_size) < 0)
+  bytes_returned = LZ4_compress_default(src, known_good_dst, src_size, max_dst_size);
+  if (bytes_returned < 1)
     run_screaming("Couldn't create a known-good destination buffer for comparison... this is bad.", 1);
-  if (LZ4_compress_default(hc_src, known_good_hc_dst, src_size, max_dst_size) < 0)
+  const size_t src_comp_size = bytes_returned;
+  bytes_returned = LZ4_compress_default(hc_src, known_good_hc_dst, src_size, max_dst_size);
+  if (bytes_returned < 1)
     run_screaming("Couldn't create a known-good (highly compressible) destination buffer for comparison... this is bad.", 1);
+  const size_t hc_src_comp_size = bytes_returned;
 
 
   /* LZ4_compress_default() */
@@ -270,19 +318,21 @@ int main(int argc, char **argv) {
   char *dst_d = calloc(1, src_size);
   memset(dst, 0, max_dst_size);
   printf("\nStarting suite A:  Normal compressible text.\n");
-  uint64_t time_taken__default       = bench(known_good_dst, ID__LZ4_COMPRESS_DEFAULT,       iterations, src,            dst,   src_size, max_dst_size);
-  uint64_t time_taken__fast          = bench(known_good_dst, ID__LZ4_COMPRESS_FAST,          iterations, src,            dst,   src_size, max_dst_size);
-  uint64_t time_taken__fast_extstate = bench(known_good_dst, ID__LZ4_COMPRESS_FAST_EXTSTATE, iterations, src,            dst,   src_size, max_dst_size);
-  //uint64_t time_taken__generic       = bench(known_good_dst, ID__LZ4_COMPRESS_GENERIC,       iterations, src,            dst,   src_size, max_dst_size);
-  uint64_t time_taken__decomp        = bench(src,            ID__LZ4_DECOMPRESS_FAST,        iterations, known_good_dst, dst_d, src_size, max_dst_size);
+  uint64_t time_taken__default       = bench(known_good_dst, ID__LZ4_COMPRESS_DEFAULT,       iterations, src,            dst,   src_size, max_dst_size, src_comp_size);
+  uint64_t time_taken__fast          = bench(known_good_dst, ID__LZ4_COMPRESS_FAST,          iterations, src,            dst,   src_size, max_dst_size, src_comp_size);
+  uint64_t time_taken__fast_extstate = bench(known_good_dst, ID__LZ4_COMPRESS_FAST_EXTSTATE, iterations, src,            dst,   src_size, max_dst_size, src_comp_size);
+  //uint64_t time_taken__generic       = bench(known_good_dst, ID__LZ4_COMPRESS_GENERIC,       iterations, src,            dst,   src_size, max_dst_size, src_comp_size);
+  uint64_t time_taken__decomp_safe   = bench(src,            ID__LZ4_DECOMPRESS_SAFE,        iterations, known_good_dst, dst_d, src_size, max_dst_size, src_comp_size);
+  uint64_t time_taken__decomp_fast   = bench(src,            ID__LZ4_DECOMPRESS_FAST,        iterations, known_good_dst, dst_d, src_size, max_dst_size, src_comp_size);
   // Suite B - Highly Compressible
   memset(dst, 0, max_dst_size);
   printf("\nStarting suite B:  Highly compressible text.\n");
-  uint64_t time_taken_hc__default       = bench(known_good_hc_dst, ID__LZ4_COMPRESS_DEFAULT,       iterations, hc_src,            dst,   src_size, max_dst_size);
-  uint64_t time_taken_hc__fast          = bench(known_good_hc_dst, ID__LZ4_COMPRESS_FAST,          iterations, hc_src,            dst,   src_size, max_dst_size);
-  uint64_t time_taken_hc__fast_extstate = bench(known_good_hc_dst, ID__LZ4_COMPRESS_FAST_EXTSTATE, iterations, hc_src,            dst,   src_size, max_dst_size);
-  //uint64_t time_taken_hc__generic       = bench(known_good_hc_dst, ID__LZ4_COMPRESS_GENERIC,       iterations, hc_src,            dst,   src_size, max_dst_size);
-  uint64_t time_taken_hc__decomp        = bench(hc_src,            ID__LZ4_DECOMPRESS_FAST,        iterations, known_good_hc_dst, dst_d, src_size, max_dst_size);
+  uint64_t time_taken_hc__default       = bench(known_good_hc_dst, ID__LZ4_COMPRESS_DEFAULT,       iterations, hc_src,            dst,   src_size, max_dst_size, hc_src_comp_size);
+  uint64_t time_taken_hc__fast          = bench(known_good_hc_dst, ID__LZ4_COMPRESS_FAST,          iterations, hc_src,            dst,   src_size, max_dst_size, hc_src_comp_size);
+  uint64_t time_taken_hc__fast_extstate = bench(known_good_hc_dst, ID__LZ4_COMPRESS_FAST_EXTSTATE, iterations, hc_src,            dst,   src_size, max_dst_size, hc_src_comp_size);
+  //uint64_t time_taken_hc__generic       = bench(known_good_hc_dst, ID__LZ4_COMPRESS_GENERIC,       iterations, hc_src,            dst,   src_size, max_dst_size, hc_src_comp_size);
+  uint64_t time_taken_hc__decomp_safe   = bench(hc_src,            ID__LZ4_DECOMPRESS_SAFE,        iterations, known_good_hc_dst, dst_d, src_size, max_dst_size, hc_src_comp_size);
+  uint64_t time_taken_hc__decomp_fast   = bench(hc_src,            ID__LZ4_DECOMPRESS_FAST,        iterations, known_good_hc_dst, dst_d, src_size, max_dst_size, hc_src_comp_size);
 
   // Report and leave.
   setlocale(LC_ALL, "");
@@ -297,13 +347,15 @@ int main(int argc, char **argv) {
   printf(format, "Normal Text", "LZ4_compress_fast()",          (double)time_taken__fast          / BILLION, (int)(iterations / ((double)time_taken__fast          /BILLION)), time_taken__fast          / iterations, (double)time_taken__fast          * 100 / time_taken__default);
   printf(format, "Normal Text", "LZ4_compress_fast_extState()", (double)time_taken__fast_extstate / BILLION, (int)(iterations / ((double)time_taken__fast_extstate /BILLION)), time_taken__fast_extstate / iterations, (double)time_taken__fast_extstate * 100 / time_taken__default);
   //printf(format, "Normal Text", "LZ4_compress_generic()",       (double)time_taken__generic       / BILLION, (int)(iterations / ((double)time_taken__generic       /BILLION)), time_taken__generic       / iterations, (double)time_taken__generic       * 100 / time_taken__default);
-  printf(format, "Normal Text", "LZ4_decompress_fast()",        (double)time_taken__decomp        / BILLION, (int)(iterations / ((double)time_taken__decomp        /BILLION)), time_taken__decomp        / iterations, (double)time_taken__decomp        * 100 / time_taken__default);
+  printf(format, "Normal Text", "LZ4_decompress_safe()",        (double)time_taken__decomp_safe   / BILLION, (int)(iterations / ((double)time_taken__decomp_safe   /BILLION)), time_taken__decomp_safe   / iterations, (double)time_taken__decomp_safe   * 100 / time_taken__default);
+  printf(format, "Normal Text", "LZ4_decompress_fast()",        (double)time_taken__decomp_fast   / BILLION, (int)(iterations / ((double)time_taken__decomp_fast   /BILLION)), time_taken__decomp_fast   / iterations, (double)time_taken__decomp_fast   * 100 / time_taken__default);
   printf(header_format, "", "", "", "", "", "");
   printf(format, "Compressible", "LZ4_compress_default()",       (double)time_taken_hc__default       / BILLION, (int)(iterations / ((double)time_taken_hc__default       /BILLION)), time_taken_hc__default       / iterations, (double)time_taken_hc__default       * 100 / time_taken_hc__default);
   printf(format, "Compressible", "LZ4_compress_fast()",          (double)time_taken_hc__fast          / BILLION, (int)(iterations / ((double)time_taken_hc__fast          /BILLION)), time_taken_hc__fast          / iterations, (double)time_taken_hc__fast          * 100 / time_taken_hc__default);
   printf(format, "Compressible", "LZ4_compress_fast_extState()", (double)time_taken_hc__fast_extstate / BILLION, (int)(iterations / ((double)time_taken_hc__fast_extstate /BILLION)), time_taken_hc__fast_extstate / iterations, (double)time_taken_hc__fast_extstate * 100 / time_taken_hc__default);
   //printf(format, "Compressible", "LZ4_compress_generic()",       (double)time_taken_hc__generic       / BILLION, (int)(iterations / ((double)time_taken_hc__generic       /BILLION)), time_taken_hc__generic       / iterations, (double)time_taken_hc__generic       * 100 / time_taken_hc__default);
-  printf(format, "Compressible", "LZ4_decompress_fast()",        (double)time_taken_hc__decomp        / BILLION, (int)(iterations / ((double)time_taken_hc__decomp        /BILLION)), time_taken_hc__decomp        / iterations, (double)time_taken_hc__decomp        * 100 / time_taken_hc__default);
+  printf(format, "Compressible", "LZ4_decompress_safe()",        (double)time_taken_hc__decomp_safe   / BILLION, (int)(iterations / ((double)time_taken_hc__decomp_safe   /BILLION)), time_taken_hc__decomp_safe   / iterations, (double)time_taken_hc__decomp_safe   * 100 / time_taken_hc__default);
+  printf(format, "Compressible", "LZ4_decompress_fast()",        (double)time_taken_hc__decomp_fast   / BILLION, (int)(iterations / ((double)time_taken_hc__decomp_fast   /BILLION)), time_taken_hc__decomp_fast   / iterations, (double)time_taken_hc__decomp_fast   * 100 / time_taken_hc__default);
   printf("%s", separator);
   printf("\n");
   printf("All done, ran %d iterations per test.\n", iterations);