From 7f96babc4bfa295ebe5abe3d2f1194f880c38769 Mon Sep 17 00:00:00 2001 From: KyleJHarper Date: Thu, 29 Oct 2015 21:47:13 -0500 Subject: Several changes to address a few concerns from Yann. See Google Group LZ4c topic 'Reusing compression/decompression resources' for details. --- examples/Makefile | 4 +- examples/basics.c | 90 ---------------------------------------- examples/compressFunctions | Bin 0 -> 68555 bytes examples/compress_functions.c | 94 ++++++++++++++++++++++++++++++++---------- examples/simple_buffer.c | 90 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 165 insertions(+), 113 deletions(-) delete mode 100644 examples/basics.c create mode 100755 examples/compressFunctions create mode 100644 examples/simple_buffer.c diff --git a/examples/Makefile b/examples/Makefile index 012c020..4075e5f 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -71,7 +71,7 @@ frameCompress: frameCompress.c compressFunctions: $(LZ4DIR)/lz4.c compress_functions.c $(CC) $(FLAGS) $^ -o $@$(EXT) -lrt -basics: $(LZ4DIR)/lz4.c basics.c +simpleBuffer: $(LZ4DIR)/lz4.c simple_buffer.c $(CC) $(FLAGS) $^ -o $@$(EXT) test : all @@ -86,6 +86,6 @@ test : all clean: @rm -f core *.o *.dec *-0 *-9 *-8192 *.lz4s *.lz4 \ printVersion$(EXT) doubleBuffer$(EXT) ringBuffer$(EXT) ringBufferHC$(EXT) \ - lineCompress$(EXT) frameCompress$(EXT) argPerformanceTesting$(EXT) + lineCompress$(EXT) frameCompress$(EXT) compressFunctions$(EXT) simpleBuffer$(EXT) @echo Cleaning completed diff --git a/examples/basics.c b/examples/basics.c deleted file mode 100644 index 79cd075..0000000 --- a/examples/basics.c +++ /dev/null @@ -1,90 +0,0 @@ -/* - * basics.c - * Copyright : Kyle Harper - * License : Follows same licensing as the lz4.c/lz4.h program at any given time. Currently, BSD 2. - * Description: Example program to demonstrate the basic usage of the compress/decompress functions within lz4.c/lz4.h. - * The functions you'll likely want are LZ4_compress_default and LZ4_decompress_fast. Both of these are documented in - * the lz4.h header file; I recommend reading them. - */ - -/* Includes, for Power! */ -#include "lz4.h" // This is all that is required to expose the prototypes for basic compression and decompression. -#include // For printf() -#include // For memcmp() -#include // For exit() - -/* - * Easy show-error-and-bail function. - */ -void run_screaming(const char *message, const int code) { - printf("%s\n", message); - exit(code); - return; -} - - -/* - * main - */ -int main(void) { - /* Introduction */ - // Below we will have a Compression and Decompression section to demonstrate. There are a few important notes before we start: - // 1) The return codes of LZ4_ functions are important. Read lz4.h if you're unsure what a given code means. - // 2) LZ4 uses char* pointers in all LZ4_ functions. This is baked into the API and probably not going to change. If your - // program uses pointers that are unsigned char*, void*, or otherwise different you may need to do some casting or set the - // right -W compiler flags to ignore those warnings (e.g.: -Wno-pointer-sign). - - /* Compression */ - // We'll store some text into a variable pointed to by *src to be compressed later. - const char *src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit."; - // The compression function needs to know how many bytes of data we're sending. The string above has 57 characters == 57 bytes. - const int src_size = 57; - // LZ4 provides a function that will tell you the maximum size of compressed output based on input data via LZ4_compressBound(). - const int max_dst_size = LZ4_compressBound(src_size); - // We will use that size for our destination boundary when allocating space. - char *compressed_data = malloc(max_dst_size); - if (compressed_data == NULL) - run_screaming("Failed to allocate memory for *compressed_data.", 1); - // That's all the information and preparation LZ4 needs to compress *src into *compressed_data. Invoke LZ4_compress_default now - // with our size values and pointers to our memory locations. Save the return value for error checking. - int return_value = 0; - return_value = LZ4_compress_default(src, compressed_data, src_size, max_dst_size); - // Check return_value to determine what happened. - if (return_value < 0) - run_screaming("A negative result from LZ4_compress_default indicates a failure trying to compress the data. See exit code (echo $?) for value returned.", return_value); - if (return_value == 0) - run_screaming("A result of 0 means compression worked, but was stopped because the destination buffer couldn't hold all the information.", 1); - if (return_value > 0) - printf("We successfully compressed some data!\n"); - // Not only does a positive return_value mean success, the value returned == the number of bytes required. You can use this to - // realloc() *compress_data to free up memory, if desired. We'll do so just to demonstrate the concept. - const int compressed_data_size = return_value; - compressed_data = (char *)realloc(compressed_data, compressed_data_size); - if (compressed_data == NULL) - run_screaming("Failed to re-alloc memory for compressed_data. Sad :(", 1); - - /* Decompression */ - // Now that we've successfully compressed the information from *src to *compressed_data, let's do the opposite! We'll create a - // *new_src location of size src_size since we know that value. - char *new_src = malloc(src_size); - if (new_src == NULL) - run_screaming("Failed to allocate memory for *new_src.", 1); - // The LZ4_decompress_fast function needs to know where the compressed data is, where the new_src memory location is, and how - // large the new_src (uncompressed) output will be. Again, save the return_value. - return_value = LZ4_decompress_fast(compressed_data, new_src, src_size); - if (return_value < 0) - run_screaming("A negative result from LZ4_decompress_fast indicates a failure trying to decompress the data. See exit code (echo $?) for value returned.", return_value); - if (return_value == 0) - run_screaming("I'm not sure this function can ever return 0. Documentation in lz4.h doesn't indicate so.", 1); - if (return_value > 0) - printf("We successfully decompressed some data!\n"); - // Not only does a positive return value mean success, the value returned == the number of bytes read from the compressed_data - // stream. I'm not sure there's ever a time you'll need to know this in most cases... - - /* Validation */ - // We should be able to compare our original *src with our *new_src and be byte-for-byte identical. - if (memcmp(src, new_src, src_size) != 0) - run_screaming("Validation failed. *src and *new_src are not identical.", 1); - printf("Validation done. The string we ended up with is:\n%s\n", new_src); - return 0; -} diff --git a/examples/compressFunctions b/examples/compressFunctions new file mode 100755 index 0000000..524bd3e Binary files /dev/null and b/examples/compressFunctions differ diff --git a/examples/compress_functions.c b/examples/compress_functions.c index 08913ef..d0dca13 100644 --- a/examples/compress_functions.c +++ b/examples/compress_functions.c @@ -5,7 +5,7 @@ * Description: A program to demonstrate the various compression functions involved in when using LZ4_compress_default(). The idea * is to show how each step in the call stack can be used directly, if desired. There is also some benchmarking for * each function to demonstrate the (probably lack of) performance difference when jumping the stack. - * (If you're new to lz4, please read basics.c to understand the fundamentals) + * (If you're new to lz4, please read simple_buffer.c to understand the fundamentals) * * The call stack (before theoretical compiler optimizations) for LZ4_compress_default is as follows: * LZ4_compress_default @@ -28,6 +28,27 @@ * directly can help avoid some test cases and branching which might be useful in some implementation-specific * situations, but you really need to know what you're doing AND what you're asking lz4 to do! You also need a * wrapper function because this function isn't exposed with lz4.h. + * + * The call stack for decompression functions is shallow. There are 2 options: + * LZ4_decompress_safe || LZ4_decompress_fast + * LZ4_decompress_generic + * + * LZ4_decompress_safe + * This is the recommended function for decompressing data. It is considered safe because the caller specifies + * both the size of the compresssed buffer to read as well as the maximum size of the output (decompressed) buffer + * instead of just the latter. + * LZ4_decompress_fast + * Again, despite its name it's not a "fast" version of decompression. It simply frees the caller of sending the + * size of the compressed buffer (it will simply be read-to-end, hence it's non-safety). + * LZ4_decompress_generic + * This is the generic function that both of the LZ4_decompress_* functions above end up calling. Calling this + * directly is not advised, period. Furthermore, it is a static inline function in lz4.c, so there isn't a symbol + * exposed for anyone using lz4.h to utilize. + * + * Special Note About Decompression: + * Using the LZ4_decompress_safe() function protects against malicious (user) input. If you are using data from a + * trusted source, or if your program is the producer (P) as well as its consumer (C) in a PC or MPMC setup, you can + * safely use the LZ4_decompress_fast function */ /* Since lz4 compiles with c99 and not gnu/std99 we need to enable POSIX linking for time.h structs and functions. */ @@ -56,7 +77,8 @@ #define ID__LZ4_COMPRESS_FAST 2 #define ID__LZ4_COMPRESS_FAST_EXTSTATE 3 #define ID__LZ4_COMPRESS_GENERIC 4 -#define ID__LZ4_DECOMPRESS_FAST 5 +#define ID__LZ4_DECOMPRESS_SAFE 5 +#define ID__LZ4_DECOMPRESS_FAST 6 @@ -84,7 +106,16 @@ void usage(const char *message) { /* * Runs the benchmark for LZ4_compress_* based on function_id. */ -uint64_t bench(const char *known_good_dst, const int function_id, const int iterations, const char *src, char *dst, const int src_size, const int max_dst_size) { +uint64_t bench( + const char *known_good_dst, + const int function_id, + const int iterations, + const char *src, + char *dst, + const size_t src_size, + const size_t max_dst_size, + const size_t comp_size + ) { uint64_t time_taken = 0; int rv = 0; const int warm_up = 5000; @@ -154,12 +185,25 @@ uint64_t bench(const char *known_good_dst, const int function_id, const int iter // } // break; + case ID__LZ4_DECOMPRESS_SAFE: + printf("Starting benchmark for function: LZ4_decompress_safe()\n"); + for(int junk=0; junk // For printf() +#include // For memcmp() +#include // For exit() + +/* + * Easy show-error-and-bail function. + */ +void run_screaming(const char *message, const int code) { + printf("%s\n", message); + exit(code); + return; +} + + +/* + * main + */ +int main(void) { + /* Introduction */ + // Below we will have a Compression and Decompression section to demonstrate. There are a few important notes before we start: + // 1) The return codes of LZ4_ functions are important. Read lz4.h if you're unsure what a given code means. + // 2) LZ4 uses char* pointers in all LZ4_ functions. This is baked into the API and probably not going to change. If your + // program uses pointers that are unsigned char*, void*, or otherwise different you may need to do some casting or set the + // right -W compiler flags to ignore those warnings (e.g.: -Wno-pointer-sign). + + /* Compression */ + // We'll store some text into a variable pointed to by *src to be compressed later. + const char *src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit."; + // The compression function needs to know how many bytes of exist. Since we're using a string, we can use strlen() + 1 (for \0). + const size_t src_size = strlen(src) + 1; + // LZ4 provides a function that will tell you the maximum size of compressed output based on input data via LZ4_compressBound(). + const size_t max_dst_size = LZ4_compressBound(src_size); + // We will use that size for our destination boundary when allocating space. + char *compressed_data = malloc(max_dst_size); + if (compressed_data == NULL) + run_screaming("Failed to allocate memory for *compressed_data.", 1); + // That's all the information and preparation LZ4 needs to compress *src into *compressed_data. Invoke LZ4_compress_default now + // with our size values and pointers to our memory locations. Save the return value for error checking. + int return_value = 0; + return_value = LZ4_compress_default(src, compressed_data, src_size, max_dst_size); + // Check return_value to determine what happened. + if (return_value < 0) + run_screaming("A negative result from LZ4_compress_default indicates a failure trying to compress the data. See exit code (echo $?) for value returned.", return_value); + if (return_value == 0) + run_screaming("A result of 0 means compression worked, but was stopped because the destination buffer couldn't hold all the information.", 1); + if (return_value > 0) + printf("We successfully compressed some data!\n"); + // Not only does a positive return_value mean success, the value returned == the number of bytes required. You can use this to + // realloc() *compress_data to free up memory, if desired. We'll do so just to demonstrate the concept. + const size_t compressed_data_size = return_value; + compressed_data = (char *)realloc(compressed_data, compressed_data_size); + if (compressed_data == NULL) + run_screaming("Failed to re-alloc memory for compressed_data. Sad :(", 1); + + /* Decompression */ + // Now that we've successfully compressed the information from *src to *compressed_data, let's do the opposite! We'll create a + // *new_src location of size src_size since we know that value. + char *new_src = malloc(src_size); + if (new_src == NULL) + run_screaming("Failed to allocate memory for *new_src.", 1); + // The LZ4_decompress_safe function needs to know where the compressed data is, how many bytes long it is, where the new_src + // memory location is, and how large the new_src (uncompressed) output will be. Again, save the return_value. + return_value = LZ4_decompress_safe(compressed_data, new_src, compressed_data_size, src_size); + if (return_value < 0) + run_screaming("A negative result from LZ4_decompress_fast indicates a failure trying to decompress the data. See exit code (echo $?) for value returned.", return_value); + if (return_value == 0) + run_screaming("I'm not sure this function can ever return 0. Documentation in lz4.h doesn't indicate so.", 1); + if (return_value > 0) + printf("We successfully decompressed some data!\n"); + // Not only does a positive return value mean success, the value returned == the number of bytes read from the compressed_data + // stream. I'm not sure there's ever a time you'll need to know this in most cases... + + /* Validation */ + // We should be able to compare our original *src with our *new_src and be byte-for-byte identical. + if (memcmp(src, new_src, src_size) != 0) + run_screaming("Validation failed. *src and *new_src are not identical.", 1); + printf("Validation done. The string we ended up with is:\n%s\n", new_src); + return 0; +} -- cgit v0.12