summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.github/workflows/ci.yml16
-rw-r--r--examples/compress_functions.c2
-rw-r--r--examples/simple_buffer.c2
-rw-r--r--examples/streaming_api_basics.md2
-rw-r--r--lib/lz4.c49
-rw-r--r--lib/lz4.h1
-rw-r--r--lib/lz4frame.h11
-rw-r--r--ossfuzz/.gitignore27
-rw-r--r--ossfuzz/decompress_fuzzer.c18
-rw-r--r--ossfuzz/round_trip_fuzzer.c65
-rw-r--r--programs/lz4cli.c72
-rw-r--r--programs/lz4io.c242
-rw-r--r--programs/util.h50
-rw-r--r--tests/.gitignore2
-rw-r--r--tests/Makefile21
-rw-r--r--tests/decompress-partial-usingDict.c88
-rw-r--r--tests/fullbench.c39
-rw-r--r--tests/fuzzer.c40
18 files changed, 550 insertions, 197 deletions
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index bf3c13e..fa7e7e9 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -253,7 +253,6 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2 # https://github.com/actions/checkout
-
- name: custom LZ4_DISTANCE_MAX; test LZ4_USER_MEMORY_FUNCTIONS
run: |
MOREFLAGS='-DLZ4_DISTANCE_MAX=8000' make V=1 check
@@ -264,6 +263,21 @@ jobs:
make V=1 clean
CC="c++ -Wno-deprecated" make V=1 -C tests fullbench-wmalloc # stricter function signature check
+ # test block device compression #1086
+ lz4cli-block-device:
+ name: Test lz4 compression on a block device
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2 # https://github.com/actions/checkout
+ - name: create a block device, compress it with lz4 # alternative : blindly use /dev/loop0, seems to always exist
+ run: |
+ make lz4
+ dd if=/dev/zero of=full0.img bs=2M count=1
+ BLOCK_DEVICE=$(sudo losetup --show -fP full0.img)
+ sudo chmod 666 $BLOCK_DEVICE
+ ./lz4 -v $BLOCK_DEVICE -c > /dev/null
+ sudo losetup -d $BLOCK_DEVICE
+ rm full0.img
###############################################################
diff --git a/examples/compress_functions.c b/examples/compress_functions.c
index e27c612..2a9d124 100644
--- a/examples/compress_functions.c
+++ b/examples/compress_functions.c
@@ -48,7 +48,7 @@
* Special Note About Decompression:
* Using the LZ4_decompress_safe() function protects against malicious (user) input. If you are using data from a
* trusted source, or if your program is the producer (P) as well as its consumer (C) in a PC or MPMC setup, you can
- * safely use the LZ4_decompress_fast function
+ * safely use the LZ4_decompress_fast function.
*/
/* Since lz4 compiles with c99 and not gnu/std99 we need to enable POSIX linking for time.h structs and functions. */
diff --git a/examples/simple_buffer.c b/examples/simple_buffer.c
index c7d59e3..f5c6eb2 100644
--- a/examples/simple_buffer.c
+++ b/examples/simple_buffer.c
@@ -47,7 +47,7 @@ int main(void) {
char* compressed_data = (char*)malloc((size_t)max_dst_size);
if (compressed_data == NULL)
run_screaming("Failed to allocate memory for *compressed_data.", 1);
- // That's all the information and preparation LZ4 needs to compress *src into *compressed_data.
+ // That's all the information and preparation LZ4 needs to compress *src into* compressed_data.
// Invoke LZ4_compress_default now with our size values and pointers to our memory locations.
// Save the return value for error checking.
const int compressed_data_size = LZ4_compress_default(src, compressed_data, src_size, max_dst_size);
diff --git a/examples/streaming_api_basics.md b/examples/streaming_api_basics.md
index 1ccc6e3..abffaef 100644
--- a/examples/streaming_api_basics.md
+++ b/examples/streaming_api_basics.md
@@ -22,7 +22,7 @@ But if you want to write advanced application, it's time to use Block or Streami
Block API (de)compresses a single contiguous memory block.
In other words, LZ4 library finds redundancy from a single contiguous memory block.
Streaming API does same thing but (de)compresses multiple adjacent contiguous memory blocks.
-So LZ4 library could find more redundancy than Block API.
+So Streaming API could find more redundancy than Block API.
The following figure shows difference between API and block sizes.
In these figures, the original data is split into 4KiBytes contiguous chunks.
diff --git a/lib/lz4.c b/lib/lz4.c
index a2272cf..7f4f175 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -700,7 +700,9 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
int LZ4_decompress_safe_forceExtDict(const char* source, char* dest,
int compressedSize, int maxOutputSize,
const void* dictStart, size_t dictSize);
-
+int LZ4_decompress_safe_partial_forceExtDict(const char* source, char* dest,
+ int compressedSize, int targetOutputSize, int dstCapacity,
+ const void* dictStart, size_t dictSize);
#if defined (__cplusplus)
}
#endif
@@ -2228,6 +2230,15 @@ int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compre
(BYTE*)dest - 64 KB, NULL, 0);
}
+LZ4_FORCE_O2
+static int LZ4_decompress_safe_partial_withPrefix64k(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity)
+{
+ dstCapacity = MIN(targetOutputSize, dstCapacity);
+ return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity,
+ endOnInputSize, partial_decode, withPrefix64k,
+ (BYTE*)dest - 64 KB, NULL, 0);
+}
+
/* Another obsolete API function, paired with the previous one. */
int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize)
{
@@ -2246,6 +2257,16 @@ static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, i
}
LZ4_FORCE_O2
+static int LZ4_decompress_safe_partial_withSmallPrefix(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity,
+ size_t prefixSize)
+{
+ dstCapacity = MIN(targetOutputSize, dstCapacity);
+ return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity,
+ endOnInputSize, partial_decode, noDict,
+ (BYTE*)dest-prefixSize, NULL, 0);
+}
+
+LZ4_FORCE_O2
int LZ4_decompress_safe_forceExtDict(const char* source, char* dest,
int compressedSize, int maxOutputSize,
const void* dictStart, size_t dictSize)
@@ -2256,6 +2277,17 @@ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest,
}
LZ4_FORCE_O2
+int LZ4_decompress_safe_partial_forceExtDict(const char* source, char* dest,
+ int compressedSize, int targetOutputSize, int dstCapacity,
+ const void* dictStart, size_t dictSize)
+{
+ dstCapacity = MIN(targetOutputSize, dstCapacity);
+ return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity,
+ endOnInputSize, partial_decode, usingExtDict,
+ (BYTE*)dest, (const BYTE*)dictStart, dictSize);
+}
+
+LZ4_FORCE_O2
static int LZ4_decompress_fast_extDict(const char* source, char* dest, int originalSize,
const void* dictStart, size_t dictSize)
{
@@ -2447,6 +2479,21 @@ int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressed
return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, (size_t)dictSize);
}
+int LZ4_decompress_safe_partial_usingDict(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity, const char* dictStart, int dictSize)
+{
+ if (dictSize==0)
+ return LZ4_decompress_safe_partial(source, dest, compressedSize, targetOutputSize, dstCapacity);
+ if (dictStart+dictSize == dest) {
+ if (dictSize >= 64 KB - 1) {
+ return LZ4_decompress_safe_partial_withPrefix64k(source, dest, compressedSize, targetOutputSize, dstCapacity);
+ }
+ assert(dictSize >= 0);
+ return LZ4_decompress_safe_partial_withSmallPrefix(source, dest, compressedSize, targetOutputSize, dstCapacity, (size_t)dictSize);
+ }
+ assert(dictSize >= 0);
+ return LZ4_decompress_safe_partial_forceExtDict(source, dest, compressedSize, targetOutputSize, dstCapacity, dictStart, (size_t)dictSize);
+}
+
int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize)
{
if (dictSize==0 || dictStart+dictSize == dest)
diff --git a/lib/lz4.h b/lib/lz4.h
index 6c068c6..07cc18e 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -430,6 +430,7 @@ LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecod
*/
LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize);
+LZ4LIB_API int LZ4_decompress_safe_partial_usingDict(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxOutputSize, const char* dictStart, int dictSize);
#endif /* LZ4_H_2983827168210 */
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 74f19cd..6bf20e4 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -243,13 +243,16 @@ typedef struct {
LZ4FLIB_API unsigned LZ4F_getVersion(void);
/*! LZ4F_createCompressionContext() :
- * The first thing to do is to create a compressionContext object, which will be used in all compression operations.
+ * The first thing to do is to create a compressionContext object,
+ * which will keep track of operation state during streaming compression.
* This is achieved using LZ4F_createCompressionContext(), which takes as argument a version.
* The version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL.
* The function will provide a pointer to a fully allocated LZ4F_cctx object.
- * If @return != zero, there was an error during context creation.
- * Object can be released using LZ4F_freeCompressionContext();
- * Note: LZ4F_freeCompressionContext() works with NULL pointers (do nothing).
+ * If @return != zero, there context creation failed.
+ * Once all streaming compression jobs are completed,
+ * the state object can be released using LZ4F_freeCompressionContext().
+ * Note1 : LZ4F_freeCompressionContext() is always successful. Its return value can be ignored.
+ * Note2 : LZ4F_freeCompressionContext() works fine with NULL input pointers (do nothing).
*/
LZ4FLIB_API LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version);
LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
diff --git a/ossfuzz/.gitignore b/ossfuzz/.gitignore
new file mode 100644
index 0000000..2dc764a
--- /dev/null
+++ b/ossfuzz/.gitignore
@@ -0,0 +1,27 @@
+# Object files
+*.o
+*.ko
+
+# Libraries
+*.lib
+*.a
+
+# Shared objects (inc. Windows DLLs)
+*.dll
+*.so
+*.so.*
+*.dylib
+*.dSYM # apple
+
+# Executables
+compress_frame_fuzzer
+compress_fuzzer
+compress_hc_fuzzer
+decompress_frame_fuzzer
+decompress_fuzzer
+round_trip_frame_fuzzer
+round_trip_fuzzer
+round_trip_hc_fuzzer
+round_trip_stream_fuzzer
+
+
diff --git a/ossfuzz/decompress_fuzzer.c b/ossfuzz/decompress_fuzzer.c
index a9a197c..490b3fd 100644
--- a/ossfuzz/decompress_fuzzer.c
+++ b/ossfuzz/decompress_fuzzer.c
@@ -49,11 +49,27 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
LZ4_decompress_safe_usingDict((char const*)dataAfterDict, dst, size,
dstCapacity, smallDict, smallDictSize);
/* Large prefix. */
- LZ4_decompress_safe_usingDict((char const*)data, dst, size,
+ LZ4_decompress_safe_usingDict((char const*)dataAfterDict, dst, size,
dstCapacity, largeDict, largeDictSize);
/* Partial decompression. */
LZ4_decompress_safe_partial((char const*)data, dst, size,
dstCapacity, dstCapacity);
+ /* Partial decompression using each possible dictionary configuration. */
+ /* Partial decompression with no dictionary. */
+ LZ4_decompress_safe_partial_usingDict((char const*)data, dst, size,
+ dstCapacity, dstCapacity, NULL, 0);
+ /* Partial decompression with small external dictionary. */
+ LZ4_decompress_safe_partial_usingDict((char const*)data, dst, size,
+ dstCapacity, dstCapacity, smallDict, smallDictSize);
+ /* Partial decompression with large external dictionary. */
+ LZ4_decompress_safe_partial_usingDict((char const*)data, dst, size,
+ dstCapacity, dstCapacity, largeDict, largeDictSize);
+ /* Partial decompression with small prefix. */
+ LZ4_decompress_safe_partial_usingDict((char const*)dataAfterDict, dst, size,
+ dstCapacity, dstCapacity, smallDict, smallDictSize);
+ /* Partial decompression wtih large prefix. */
+ LZ4_decompress_safe_partial_usingDict((char const*)dataAfterDict, dst, size,
+ dstCapacity, dstCapacity, largeDict, largeDictSize);
free(dst);
free(dict);
FUZZ_dataProducer_free(producer);
diff --git a/ossfuzz/round_trip_fuzzer.c b/ossfuzz/round_trip_fuzzer.c
index 6307058..7a2f768 100644
--- a/ossfuzz/round_trip_fuzzer.c
+++ b/ossfuzz/round_trip_fuzzer.c
@@ -20,8 +20,13 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
size_t const partialCapacity = FUZZ_getRange_from_uint32(partialCapacitySeed, 0, size);
size_t const dstCapacity = LZ4_compressBound(size);
-
- char* const dst = (char*)malloc(dstCapacity);
+ size_t const largeSize = 64 * 1024 - 1;
+ size_t const smallSize = 1024;
+ char* const dstPlusLargePrefix = (char*)malloc(dstCapacity + largeSize);
+ char* const dstPlusSmallPrefix = dstPlusLargePrefix + largeSize - smallSize;
+ char* const largeDict = (char*)malloc(largeSize);
+ char* const smallDict = largeDict + largeSize - smallSize;
+ char* const dst = dstPlusLargePrefix + largeSize;
char* const rt = (char*)malloc(size);
FUZZ_ASSERT(dst);
@@ -47,7 +52,61 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!");
free(partial);
}
-
+ /* Partial decompression using dict with no dict. */
+ {
+ char* const partial = (char*)malloc(partialCapacity);
+ FUZZ_ASSERT(partial);
+ int const partialSize = LZ4_decompress_safe_partial_usingDict(
+ dst, partial, dstSize, partialCapacity, partialCapacity, NULL, 0);
+ FUZZ_ASSERT(partialSize >= 0);
+ FUZZ_ASSERT_MSG(partialSize == partialCapacity, "Incorrect size");
+ FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!");
+ free(partial);
+ }
+ /* Partial decompression using dict with small prefix as dict */
+ {
+ char* const partial = (char*)malloc(partialCapacity);
+ FUZZ_ASSERT(partial);
+ int const partialSize = LZ4_decompress_safe_partial_usingDict(
+ dst, partial, dstSize, partialCapacity, partialCapacity, dstPlusSmallPrefix, smallSize);
+ FUZZ_ASSERT(partialSize >= 0);
+ FUZZ_ASSERT_MSG(partialSize == partialCapacity, "Incorrect size");
+ FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!");
+ free(partial);
+ }
+ /* Partial decompression using dict with large prefix as dict */
+ {
+ char* const partial = (char*)malloc(partialCapacity);
+ FUZZ_ASSERT(partial);
+ int const partialSize = LZ4_decompress_safe_partial_usingDict(
+ dst, partial, dstSize, partialCapacity, partialCapacity, dstPlusLargePrefix, largeSize);
+ FUZZ_ASSERT(partialSize >= 0);
+ FUZZ_ASSERT_MSG(partialSize == partialCapacity, "Incorrect size");
+ FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!");
+ free(partial);
+ }
+ /* Partial decompression using dict with small external dict */
+ {
+ char* const partial = (char*)malloc(partialCapacity);
+ FUZZ_ASSERT(partial);
+ int const partialSize = LZ4_decompress_safe_partial_usingDict(
+ dst, partial, dstSize, partialCapacity, partialCapacity, smallDict, smallSize);
+ FUZZ_ASSERT(partialSize >= 0);
+ FUZZ_ASSERT_MSG(partialSize == partialCapacity, "Incorrect size");
+ FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!");
+ free(partial);
+ }
+ /* Partial decompression using dict with large external dict */
+ {
+ char* const partial = (char*)malloc(partialCapacity);
+ FUZZ_ASSERT(partial);
+ int const partialSize = LZ4_decompress_safe_partial_usingDict(
+ dst, partial, dstSize, partialCapacity, partialCapacity, largeDict, largeSize);
+ FUZZ_ASSERT(partialSize >= 0);
+ FUZZ_ASSERT_MSG(partialSize == partialCapacity, "Incorrect size");
+ FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!");
+ free(partial);
+ }
free(dst);
free(rt);
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index b5cb000..254a6ce 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -314,6 +314,7 @@ int main(int argc, const char** argv)
cLevelLast=-10000,
legacy_format=0,
forceStdout=0,
+ forceOverwrite=0,
main_pause=0,
multiple_inputs=0,
all_arguments_are_files=0,
@@ -330,9 +331,8 @@ int main(int argc, const char** argv)
const char extension[] = LZ4_EXTENSION;
size_t blockSize = LZ4IO_setBlockSizeID(prefs, LZ4_BLOCKSIZEID_DEFAULT);
const char* const exeName = lastNameFromPath(argv[0]);
-#ifdef UTIL_HAS_CREATEFILELIST
- const char** extendedFileList = NULL;
char* fileNamesBuf = NULL;
+#ifdef UTIL_HAS_CREATEFILELIST
unsigned fileNamesNb, recursive=0;
#endif
@@ -491,7 +491,7 @@ int main(int argc, const char** argv)
case 't': mode = om_test; break;
/* Overwrite */
- case 'f': LZ4IO_setOverwrite(prefs, 1); break;
+ case 'f': forceOverwrite=1; LZ4IO_setOverwrite(prefs, 1); break;
/* Verbose mode */
case 'v': displayLevel++; break;
@@ -581,20 +581,24 @@ int main(int argc, const char** argv)
}
/* Store in *inFileNames[] if -m is used. */
- if (multiple_inputs) { inFileNames[ifnIdx++]=argument; continue; }
+ if (multiple_inputs) { inFileNames[ifnIdx++] = argument; continue; }
- /* Store first non-option arg in input_filename to preserve original cli logic. */
- if (!input_filename) { input_filename=argument; continue; }
+ /* original cli logic : lz4 input output */
+ /* First non-option arg is input_filename. */
+ if (!input_filename) { input_filename = argument; continue; }
- /* Second non-option arg in output_filename to preserve original cli logic. */
+ /* Second non-option arg is output_filename */
if (!output_filename) {
- output_filename=argument;
+ output_filename = argument;
if (!strcmp (output_filename, nullOutput)) output_filename = nulmark;
continue;
}
- /* 3rd non-option arg should not exist */
- DISPLAYLEVEL(1, "Warning : %s won't be used ! Do you want multiple input files (-m) ? \n", argument);
+ /* 3rd+ non-option arg should not exist */
+ DISPLAYLEVEL(1, "%s : %s won't be used ! Do you want multiple input files (-m) ? \n",
+ forceOverwrite ? "Warning" : "Error",
+ argument);
+ if (!forceOverwrite) exit(1);
}
DISPLAYLEVEL(3, WELCOME_MESSAGE);
@@ -617,7 +621,7 @@ int main(int argc, const char** argv)
input_filename = inFileNames[0];
#ifdef UTIL_HAS_CREATEFILELIST
if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */
- extendedFileList = UTIL_createFileList(inFileNames, ifnIdx, &fileNamesBuf, &fileNamesNb);
+ const char** extendedFileList = UTIL_createFileList(inFileNames, ifnIdx, &fileNamesBuf, &fileNamesNb);
if (extendedFileList) {
unsigned u;
for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, extendedFileList[u]);
@@ -649,27 +653,19 @@ int main(int argc, const char** argv)
mode = om_decompress; /* defer to decompress */
}
- /* compress or decompress */
+ /* No input provided => use stdin */
if (!input_filename) input_filename = stdinmark;
- /* Check if input is defined as console; trigger an error in this case */
+
+ /* Refuse to use the console as input */
if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) {
DISPLAYLEVEL(1, "refusing to read from a console\n");
exit(1);
}
+
if (!strcmp(input_filename, stdinmark)) {
/* if input==stdin and no output defined, stdout becomes default output */
if (!output_filename) output_filename = stdoutmark;
}
- else{
-#ifdef UTIL_HAS_CREATEFILELIST
- if (!recursive && !UTIL_isRegFile(input_filename)) {
-#else
- if (!UTIL_isRegFile(input_filename)) {
-#endif
- DISPLAYLEVEL(1, "%s: is not a regular file \n", input_filename);
- exit(1);
- }
- }
/* No output filename ==> try to select one automatically (when possible) */
while ((!output_filename) && (multiple_inputs==0)) {
@@ -679,7 +675,7 @@ int main(int argc, const char** argv)
* To ensure `stdout` is explicitly selected, use `-c` command flag.
* Conversely, to ensure output will not become `stdout`, use `-m` command flag */
DISPLAYLEVEL(1, "Warning : using stdout as default output. Do not rely on this behavior: use explicit `-c` instead ! \n");
- output_filename=stdoutmark;
+ output_filename = stdoutmark;
break;
}
if (mode == om_auto) { /* auto-determine compression or decompression, based on file extension */
@@ -695,7 +691,7 @@ int main(int argc, const char** argv)
DISPLAYLEVEL(2, "Compressed filename will be : %s \n", output_filename);
break;
}
- if (mode == om_decompress) {/* decompression to file (automatic name will work only if input filename has correct format extension) */
+ if (mode == om_decompress) {/* decompress to file (automatic output name only works if input filename has correct format extension) */
size_t outl;
size_t const inl = strlen(input_filename);
dynNameSpace = (char*)calloc(1,inl+1);
@@ -704,20 +700,17 @@ int main(int argc, const char** argv)
outl = inl;
if (inl>4)
while ((outl >= inl-4) && (input_filename[outl] == extension[outl-inl+4])) dynNameSpace[outl--]=0;
- if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); badusage(exeName); }
+ if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename \n"); badusage(exeName); }
output_filename = dynNameSpace;
DISPLAYLEVEL(2, "Decoding file %s \n", output_filename);
}
break;
}
- if (mode == om_list){
- if(!multiple_inputs){
- inFileNames[ifnIdx++] = input_filename;
- }
- }
- else{
- if (multiple_inputs==0) assert(output_filename);
+ if (mode == om_list) {
+ if (!multiple_inputs) inFileNames[ifnIdx++] = input_filename;
+ } else {
+ if (!multiple_inputs) assert(output_filename != NULL);
}
/* when multiple_inputs==1, output_filename may simply be useless,
* however, output_filename must be !NULL for next strcmp() tests */
@@ -745,8 +738,10 @@ int main(int argc, const char** argv)
if (ifnIdx == 0) multiple_inputs = 0;
if (mode == om_decompress) {
if (multiple_inputs) {
- const char* const dec_extension = !strcmp(output_filename,stdoutmark) ? stdoutmark : LZ4_EXTENSION;
- assert(ifnIdx <= INT_MAX);
+ const char* dec_extension = LZ4_EXTENSION;
+ if (!strcmp(output_filename, stdoutmark)) dec_extension = stdoutmark;
+ if (!strcmp(output_filename, nulmark)) dec_extension = nulmark;
+ assert(ifnIdx < INT_MAX);
operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, (int)ifnIdx, dec_extension, prefs);
} else {
operationResult = DEFAULT_DECOMPRESSOR(input_filename, output_filename, prefs);
@@ -774,12 +769,7 @@ int main(int argc, const char** argv)
_cleanup:
if (main_pause) waitEnter();
free(dynNameSpace);
-#ifdef UTIL_HAS_CREATEFILELIST
- if (extendedFileList) {
- UTIL_freeFileList(extendedFileList, fileNamesBuf);
- inFileNames = NULL;
- }
-#endif
+ free(fileNamesBuf);
LZ4IO_freePreferences(prefs);
free((void*)inFileNames);
return operationResult;
diff --git a/programs/lz4io.c b/programs/lz4io.c
index 6f636b5..36736c2 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -103,29 +103,9 @@ static int g_displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result
} }
static const clock_t refreshRate = CLOCKS_PER_SEC / 6;
static clock_t g_time = 0;
-#define LZ4IO_STATIC_ASSERT(c) { enum { LZ4IO_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */
-
-/**************************************
-* Local Parameters
-**************************************/
+#define LZ4IO_STATIC_ASSERT(c) { enum { LZ4IO_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */
-struct LZ4IO_prefs_s {
- int passThrough;
- int overwrite;
- int testMode;
- int blockSizeId;
- size_t blockSize;
- int blockChecksum;
- int streamChecksum;
- int blockIndependence;
- int sparseFileSupport;
- int contentSizeFlag;
- int useDictionary;
- unsigned favorDecSpeed;
- const char* dictionaryFilename;
- int removeSrcFile;
-};
/**************************************
* Exceptions
@@ -134,7 +114,7 @@ struct LZ4IO_prefs_s {
# define DEBUG 0
#endif
#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
-#define EXM_THROW(error, ...) \
+#define END_PROCESS(error, ...) \
{ \
DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
DISPLAYLEVEL(1, "Error %i : ", error); \
@@ -144,20 +124,31 @@ struct LZ4IO_prefs_s {
}
-/**************************************
-* Version modifiers
-**************************************/
-#define DEFAULT_DECOMPRESSOR LZ4IO_decompressLZ4F
-
-
/* ************************************************** */
/* ****************** Parameters ******************** */
/* ************************************************** */
+struct LZ4IO_prefs_s {
+ int passThrough;
+ int overwrite;
+ int testMode;
+ int blockSizeId;
+ size_t blockSize;
+ int blockChecksum;
+ int streamChecksum;
+ int blockIndependence;
+ int sparseFileSupport;
+ int contentSizeFlag;
+ int useDictionary;
+ unsigned favorDecSpeed;
+ const char* dictionaryFilename;
+ int removeSrcFile;
+};
+
LZ4IO_prefs_t* LZ4IO_defaultPreferences(void)
{
LZ4IO_prefs_t* const ret = (LZ4IO_prefs_t*)malloc(sizeof(*ret));
- if (!ret) EXM_THROW(21, "Allocation error : not enough memory");
+ if (!ret) END_PROCESS(21, "Allocation error : not enough memory");
ret->passThrough = 0;
ret->overwrite = 1;
ret->testMode = 0;
@@ -295,6 +286,26 @@ void LZ4IO_setRemoveSrcFile(LZ4IO_prefs_t* const prefs, unsigned flag)
/* ************************************************************************ **
+** ********************** String functions ********************* **
+** ************************************************************************ */
+
+static int LZ4IO_isDevNull(const char* s)
+{
+ return UTIL_sameString(s, nulmark);
+}
+
+static int LZ4IO_isStdin(const char* s)
+{
+ return UTIL_sameString(s, stdinmark);
+}
+
+static int LZ4IO_isStdout(const char* s)
+{
+ return UTIL_sameString(s, stdoutmark);
+}
+
+
+/* ************************************************************************ **
** ********************** LZ4 File / Pipe compression ********************* **
** ************************************************************************ */
@@ -310,13 +321,13 @@ static FILE* LZ4IO_openSrcFile(const char* srcFileName)
{
FILE* f;
- if (!strcmp (srcFileName, stdinmark)) {
- DISPLAYLEVEL(4,"Using stdin for input\n");
+ if (LZ4IO_isStdin(srcFileName)) {
+ DISPLAYLEVEL(4,"Using stdin for input \n");
f = stdin;
SET_BINARY_MODE(stdin);
} else {
f = fopen(srcFileName, "rb");
- if ( f==NULL ) DISPLAYLEVEL(1, "%s: %s \n", srcFileName, strerror(errno));
+ if (f==NULL) DISPLAYLEVEL(1, "%s: %s \n", srcFileName, strerror(errno));
}
return f;
@@ -331,7 +342,7 @@ static FILE* LZ4IO_openDstFile(const char* dstFileName, const LZ4IO_prefs_t* con
FILE* f;
assert(dstFileName != NULL);
- if (!strcmp (dstFileName, stdoutmark)) {
+ if (LZ4IO_isStdout(dstFileName)) {
DISPLAYLEVEL(4, "Using stdout for output \n");
f = stdout;
SET_BINARY_MODE(stdout);
@@ -340,7 +351,8 @@ static FILE* LZ4IO_openDstFile(const char* dstFileName, const LZ4IO_prefs_t* con
" to force-enable it, add --sparse command \n");
}
} else {
- if (!prefs->overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */
+ if (!prefs->overwrite && !LZ4IO_isDevNull(dstFileName)) {
+ /* Check if destination file already exists */
FILE* const testf = fopen( dstFileName, "rb" );
if (testf != NULL) { /* dest exists, prompt for overwrite authorization */
fclose(testf);
@@ -348,7 +360,7 @@ static FILE* LZ4IO_openDstFile(const char* dstFileName, const LZ4IO_prefs_t* con
DISPLAY("%s already exists; not overwritten \n", dstFileName);
return NULL;
}
- DISPLAY("%s already exists; do you wish to overwrite (y/N) ? ", dstFileName);
+ DISPLAY("%s already exists; do you want to overwrite (y/N) ? ", dstFileName);
{ int ch = getchar();
if ((ch!='Y') && (ch!='y')) {
DISPLAY(" not overwritten \n");
@@ -414,24 +426,24 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
/* Init */
clock_t const clockStart = clock();
if (finput == NULL)
- EXM_THROW(20, "%s : open file error ", input_filename);
+ END_PROCESS(20, "%s : open file error ", input_filename);
foutput = LZ4IO_openDstFile(output_filename, prefs);
if (foutput == NULL) {
fclose(finput);
- EXM_THROW(20, "%s : open file error ", input_filename);
+ END_PROCESS(20, "%s : open file error ", input_filename);
}
/* Allocate Memory */
in_buff = (char*)malloc(LEGACY_BLOCKSIZE);
out_buff = (char*)malloc((size_t)outBuffSize + 4);
if (!in_buff || !out_buff)
- EXM_THROW(21, "Allocation error : not enough memory");
+ END_PROCESS(21, "Allocation error : not enough memory");
/* Write Archive Header */
LZ4IO_writeLE32(out_buff, LEGACY_MAGICNUMBER);
if (fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput) != MAGICNUMBER_SIZE)
- EXM_THROW(22, "Write error : cannot write header");
+ END_PROCESS(22, "Write error : cannot write header");
/* Main Loop */
while (1) {
@@ -454,13 +466,13 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
assert(outSize < outBuffSize);
LZ4IO_writeLE32(out_buff, (unsigned)outSize);
if (fwrite(out_buff, 1, (size_t)outSize+4, foutput) != (size_t)(outSize+4)) {
- EXM_THROW(24, "Write error : cannot write compressed block");
+ END_PROCESS(24, "Write error : cannot write compressed block");
} }
- if (ferror(finput)) EXM_THROW(25, "Error while reading %s ", input_filename);
+ if (ferror(finput)) END_PROCESS(24, "Error while reading %s ", input_filename);
/* Status */
clockEnd = clock();
- if (clockEnd==clockStart) clockEnd+=1; /* avoid division by zero (speed) */
+ clockEnd += (clockEnd==clockStart); /* avoid division by zero (speed) */
filesize += !filesize; /* avoid division by zero (ratio) */
DISPLAYLEVEL(2, "\r%79s\r", ""); /* blank line */
DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
@@ -474,7 +486,7 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
free(in_buff);
free(out_buff);
fclose(finput);
- if (strcmp(output_filename,stdoutmark)) fclose(foutput); /* do not close stdout */
+ if (!LZ4IO_isStdout(output_filename)) fclose(foutput); /* do not close stdout */
return 0;
}
@@ -499,7 +511,7 @@ int LZ4IO_compressMultipleFilenames_Legacy(
/* loop on each file */
for (i=0; i<ifntSize; i++) {
size_t const ifnSize = strlen(inFileNamesTable[i]);
- if (!strcmp(suffix, stdoutmark)) {
+ if (LZ4IO_isStdout(suffix)) {
missed_files += LZ4IO_compressFilename_Legacy(
inFileNamesTable[i], stdoutmark,
compressionLevel, prefs);
@@ -531,7 +543,6 @@ int LZ4IO_compressMultipleFilenames_Legacy(
/*********************************************
* Compression using Frame format
*********************************************/
-
typedef struct {
void* srcBuffer;
size_t srcBufferSize;
@@ -552,15 +563,15 @@ static void* LZ4IO_createDict(size_t* dictSize, const char* const dictFilename)
char* dictBuf;
FILE* dictFile;
- if (!circularBuf) EXM_THROW(25, "Allocation error : not enough memory for circular buffer");
- if (!dictFilename) EXM_THROW(25, "Dictionary error : no filename provided");
+ if (!circularBuf) END_PROCESS(25, "Allocation error : not enough memory for circular buffer");
+ if (!dictFilename) END_PROCESS(26, "Dictionary error : no filename provided");
dictFile = LZ4IO_openSrcFile(dictFilename);
- if (!dictFile) EXM_THROW(25, "Dictionary error : could not open dictionary file");
+ if (!dictFile) END_PROCESS(27, "Dictionary error : could not open dictionary file");
- /* opportunistically seek to the part of the file we care about. If this */
- /* fails it's not a problem since we'll just read everything anyways. */
- if (strcmp(dictFilename, stdinmark)) {
+ /* opportunistically seek to the part of the file we care about.
+ * If this fails it's not a problem since we'll just read everything anyways. */
+ if (!LZ4IO_isStdin(dictFilename)) {
(void)UTIL_fseek(dictFile, -LZ4_MAX_DICT_SIZE, SEEK_END);
}
@@ -585,7 +596,7 @@ static void* LZ4IO_createDict(size_t* dictSize, const char* const dictFilename)
} else {
/* Otherwise, we will alloc a new buffer and copy our dict into that. */
dictBuf = (char *)malloc(dictLen ? dictLen : 1);
- if (!dictBuf) EXM_THROW(25, "Allocation error : not enough memory");
+ if (!dictBuf) END_PROCESS(28, "Allocation error : not enough memory");
memcpy(dictBuf, circularBuf + dictStart, circularBufSize - dictStart);
memcpy(dictBuf + circularBufSize - dictStart, circularBuf, dictLen - (circularBufSize - dictStart));
@@ -604,7 +615,7 @@ static LZ4F_CDict* LZ4IO_createCDict(const LZ4IO_prefs_t* const prefs)
LZ4F_CDict* cdict;
if (!prefs->useDictionary) return NULL;
dictionaryBuffer = LZ4IO_createDict(&dictionarySize, prefs->dictionaryFilename);
- if (!dictionaryBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary");
+ if (!dictionaryBuffer) END_PROCESS(29, "Dictionary error : could not create dictionary");
cdict = LZ4F_createCDict(dictionaryBuffer, dictionarySize);
free(dictionaryBuffer);
return cdict;
@@ -616,14 +627,14 @@ static cRess_t LZ4IO_createCResources(const LZ4IO_prefs_t* const prefs)
cRess_t ress;
LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&(ress.ctx), LZ4F_VERSION);
- if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode));
+ if (LZ4F_isError(errorCode)) END_PROCESS(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode));
/* Allocate Memory */
ress.srcBuffer = malloc(blockSize);
ress.srcBufferSize = blockSize;
ress.dstBufferSize = LZ4F_compressFrameBound(blockSize, NULL); /* cover worst case */
ress.dstBuffer = malloc(ress.dstBufferSize);
- if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory");
+ if (!ress.srcBuffer || !ress.dstBuffer) END_PROCESS(31, "Allocation error : not enough memory");
ress.cdict = LZ4IO_createCDict(prefs);
@@ -639,7 +650,7 @@ static void LZ4IO_freeCResources(cRess_t ress)
ress.cdict = NULL;
{ LZ4F_errorCode_t const errorCode = LZ4F_freeCompressionContext(ress.ctx);
- if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); }
+ if (LZ4F_isError(errorCode)) END_PROCESS(35, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); }
}
/*
@@ -687,7 +698,7 @@ LZ4IO_compressFilename_extRess(cRess_t ress,
/* read first block */
readSize = fread(srcBuffer, (size_t)1, blockSize, srcFile);
- if (ferror(srcFile)) EXM_THROW(30, "Error reading %s ", srcFileName);
+ if (ferror(srcFile)) END_PROCESS(40, "Error reading %s ", srcFileName);
filesize += readSize;
/* single-block file */
@@ -695,14 +706,14 @@ LZ4IO_compressFilename_extRess(cRess_t ress,
/* Compress in single pass */
size_t const cSize = LZ4F_compressFrame_usingCDict(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs);
if (LZ4F_isError(cSize))
- EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize));
+ END_PROCESS(41, "Compression failed : %s", LZ4F_getErrorName(cSize));
compressedfilesize = cSize;
DISPLAYUPDATE(2, "\rRead : %u MiB ==> %.2f%% ",
(unsigned)(filesize>>20), (double)compressedfilesize/(filesize+!filesize)*100); /* avoid division by zero */
/* Write Block */
if (fwrite(dstBuffer, 1, cSize, dstFile) != cSize) {
- EXM_THROW(32, "Write error : failed writing single-block compressed frame");
+ END_PROCESS(42, "Write error : failed writing single-block compressed frame");
} }
else
@@ -711,55 +722,55 @@ LZ4IO_compressFilename_extRess(cRess_t ress,
{
/* Write Frame Header */
size_t const headerSize = LZ4F_compressBegin_usingCDict(ctx, dstBuffer, dstBufferSize, ress.cdict, &prefs);
- if (LZ4F_isError(headerSize)) EXM_THROW(33, "File header generation failed : %s", LZ4F_getErrorName(headerSize));
+ if (LZ4F_isError(headerSize)) END_PROCESS(43, "File header generation failed : %s", LZ4F_getErrorName(headerSize));
if (fwrite(dstBuffer, 1, headerSize, dstFile) != headerSize)
- EXM_THROW(34, "Write error : cannot write header");
+ END_PROCESS(44, "Write error : cannot write header");
compressedfilesize += headerSize;
/* Main Loop - one block at a time */
while (readSize>0) {
size_t const outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL);
if (LZ4F_isError(outSize))
- EXM_THROW(35, "Compression failed : %s", LZ4F_getErrorName(outSize));
+ END_PROCESS(45, "Compression failed : %s", LZ4F_getErrorName(outSize));
compressedfilesize += outSize;
DISPLAYUPDATE(2, "\rRead : %u MiB ==> %.2f%% ",
(unsigned)(filesize>>20), (double)compressedfilesize/filesize*100);
/* Write Block */
if (fwrite(dstBuffer, 1, outSize, dstFile) != outSize)
- EXM_THROW(36, "Write error : cannot write compressed block");
+ END_PROCESS(46, "Write error : cannot write compressed block");
/* Read next block */
readSize = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile);
filesize += readSize;
}
- if (ferror(srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName);
+ if (ferror(srcFile)) END_PROCESS(47, "Error reading %s ", srcFileName);
/* End of Frame mark */
{ size_t const endSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL);
if (LZ4F_isError(endSize))
- EXM_THROW(38, "End of frame error : %s", LZ4F_getErrorName(endSize));
+ END_PROCESS(48, "End of frame error : %s", LZ4F_getErrorName(endSize));
if (fwrite(dstBuffer, 1, endSize, dstFile) != endSize)
- EXM_THROW(39, "Write error : cannot write end of frame");
+ END_PROCESS(49, "Write error : cannot write end of frame");
compressedfilesize += endSize;
} }
/* Release file handlers */
fclose (srcFile);
- if (strcmp(dstFileName,stdoutmark)) fclose (dstFile); /* do not close stdout */
+ if (!LZ4IO_isStdout(dstFileName)) fclose(dstFile); /* do not close stdout */
/* Copy owner, file permissions and modification time */
{ stat_t statbuf;
- if (strcmp (srcFileName, stdinmark)
- && strcmp (dstFileName, stdoutmark)
- && strcmp (dstFileName, nulmark)
+ if (!LZ4IO_isStdin(srcFileName)
+ && !LZ4IO_isStdout(dstFileName)
+ && !LZ4IO_isDevNull(dstFileName)
&& UTIL_getFileStat(srcFileName, &statbuf)) {
UTIL_setFileStat(dstFileName, &statbuf);
} }
if (io_prefs->removeSrcFile) { /* remove source file : --rm */
if (remove(srcFileName))
- EXM_THROW(40, "Remove error : %s: %s", srcFileName, strerror(errno));
+ END_PROCESS(50, "Remove error : %s: %s", srcFileName, strerror(errno));
}
/* Final Status */
@@ -815,12 +826,13 @@ int LZ4IO_compressMultipleFilenames(
/* loop on each file */
for (i=0; i<ifntSize; i++) {
size_t const ifnSize = strlen(inFileNamesTable[i]);
- if (!strcmp(suffix, stdoutmark)) {
+ if (LZ4IO_isStdout(suffix)) {
missed_files += LZ4IO_compressFilename_extRess(ress,
inFileNamesTable[i], stdoutmark,
compressionLevel, prefs);
continue;
}
+ /* suffix != stdout => compress into a file => generate its name */
if (ofnSize <= ifnSize+suffixSize+1) {
free(dstFileName);
ofnSize = ifnSize + 20;
@@ -878,14 +890,14 @@ LZ4IO_fwriteSparse(FILE* file,
if (!sparseMode) { /* normal write */
size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file);
- if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : cannot write decoded block");
+ if (sizeCheck != bufferSize) END_PROCESS(70, "Write error : cannot write decoded block");
return 0;
}
/* avoid int overflow */
if (storedSkips > 1 GB) {
int const seekResult = UTIL_fseek(file, 1 GB, SEEK_CUR);
- if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)");
+ if (seekResult != 0) END_PROCESS(71, "1 GB skip error (sparse file support)");
storedSkips -= 1 GB;
}
@@ -902,13 +914,13 @@ LZ4IO_fwriteSparse(FILE* file,
if (nb0T != seg0SizeT) { /* not all 0s */
errno = 0;
{ int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR);
- if (seekResult) EXM_THROW(72, "Sparse skip error(%d): %s ; try --no-sparse", (int)errno, strerror(errno));
+ if (seekResult) END_PROCESS(72, "Sparse skip error(%d): %s ; try --no-sparse", (int)errno, strerror(errno));
}
storedSkips = 0;
seg0SizeT -= nb0T;
ptrT += nb0T;
{ size_t const sizeCheck = fwrite(ptrT, sizeT, seg0SizeT, file);
- if (sizeCheck != seg0SizeT) EXM_THROW(73, "Write error : cannot write decoded block");
+ if (sizeCheck != seg0SizeT) END_PROCESS(73, "Write error : cannot write decoded block");
} }
ptrT += seg0SizeT;
}
@@ -922,10 +934,10 @@ LZ4IO_fwriteSparse(FILE* file,
storedSkips += (unsigned) (restPtr - restStart);
if (restPtr != restEnd) {
int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR);
- if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse");
+ if (seekResult) END_PROCESS(74, "Sparse skip error ; try --no-sparse");
storedSkips = 0;
{ size_t const sizeCheck = fwrite(restPtr, 1, (size_t)(restEnd - restPtr), file);
- if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(75, "Write error : cannot write decoded end of block");
+ if (sizeCheck != (size_t)(restEnd - restPtr)) END_PROCESS(75, "Write error : cannot write decoded end of block");
} }
}
@@ -937,9 +949,9 @@ static void LZ4IO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
if (storedSkips>0) { /* implies sparseFileSupport>0 */
const char lastZeroByte[1] = { 0 };
if (UTIL_fseek(file, storedSkips-1, SEEK_CUR) != 0)
- EXM_THROW(69, "Final skip error (sparse file)\n");
+ END_PROCESS(68, "Final skip error (sparse file)\n");
if (fwrite(lastZeroByte, 1, 1, file) != 1)
- EXM_THROW(69, "Write error : cannot write last zero\n");
+ END_PROCESS(69, "Write error : cannot write last zero\n");
}
}
@@ -955,7 +967,7 @@ LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput, const LZ4IO_prefs_t* prefs
/* Allocate Memory */
char* const in_buff = (char*)malloc((size_t)LZ4_compressBound(LEGACY_BLOCKSIZE));
char* const out_buff = (char*)malloc(LEGACY_BLOCKSIZE);
- if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory");
+ if (!in_buff || !out_buff) END_PROCESS(51, "Allocation error : not enough memory");
/* Main Loop */
while (1) {
@@ -964,7 +976,7 @@ LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput, const LZ4IO_prefs_t* prefs
/* Block Size */
{ size_t const sizeCheck = fread(in_buff, 1, LZ4IO_LEGACY_BLOCK_HEADER_SIZE, finput);
if (sizeCheck == 0) break; /* Nothing to read : file read is completed */
- if (sizeCheck != LZ4IO_LEGACY_BLOCK_HEADER_SIZE) EXM_THROW(52, "Read error : cannot access block size ");
+ if (sizeCheck != LZ4IO_LEGACY_BLOCK_HEADER_SIZE) END_PROCESS(52, "Read error : cannot access block size ");
}
blockSize = LZ4IO_readLE32(in_buff); /* Convert to Little Endian */
if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) {
@@ -975,16 +987,16 @@ LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput, const LZ4IO_prefs_t* prefs
/* Read Block */
{ size_t const sizeCheck = fread(in_buff, 1, blockSize, finput);
- if (sizeCheck != blockSize) EXM_THROW(52, "Read error : cannot access compressed block !"); }
+ if (sizeCheck != blockSize) END_PROCESS(53, "Read error : cannot access compressed block !"); }
/* Decode Block */
{ int const decodeSize = LZ4_decompress_safe(in_buff, out_buff, (int)blockSize, LEGACY_BLOCKSIZE);
- if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !");
+ if (decodeSize < 0) END_PROCESS(54, "Decoding Failed ! Corrupted input detected !");
streamSize += (unsigned long long)decodeSize;
/* Write Block */
storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, (size_t)decodeSize, prefs->sparseFileSupport, storedSkips); /* success or die */
} }
- if (ferror(finput)) EXM_THROW(54, "Read error : ferror");
+ if (ferror(finput)) END_PROCESS(55, "Read error : ferror");
LZ4IO_fwriteSparseEnd(foutput, storedSkips);
@@ -1017,7 +1029,7 @@ static void LZ4IO_loadDDict(dRess_t* ress, const LZ4IO_prefs_t* const prefs)
}
ress->dictBuffer = LZ4IO_createDict(&ress->dictBufferSize, prefs->dictionaryFilename);
- if (!ress->dictBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary");
+ if (!ress->dictBuffer) END_PROCESS(25, "Dictionary error : could not create dictionary");
}
static const size_t LZ4IO_dBufferSize = 64 KB;
@@ -1027,14 +1039,14 @@ static dRess_t LZ4IO_createDResources(const LZ4IO_prefs_t* const prefs)
/* init */
LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&ress.dCtx, LZ4F_VERSION);
- if (LZ4F_isError(errorCode)) EXM_THROW(60, "Can't create LZ4F context : %s", LZ4F_getErrorName(errorCode));
+ if (LZ4F_isError(errorCode)) END_PROCESS(60, "Can't create LZ4F context : %s", LZ4F_getErrorName(errorCode));
/* Allocate Memory */
ress.srcBufferSize = LZ4IO_dBufferSize;
ress.srcBuffer = malloc(ress.srcBufferSize);
ress.dstBufferSize = LZ4IO_dBufferSize;
ress.dstBuffer = malloc(ress.dstBufferSize);
- if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
+ if (!ress.srcBuffer || !ress.dstBuffer) END_PROCESS(61, "Allocation error : not enough memory");
LZ4IO_loadDDict(&ress, prefs);
@@ -1045,7 +1057,7 @@ static dRess_t LZ4IO_createDResources(const LZ4IO_prefs_t* const prefs)
static void LZ4IO_freeDResources(dRess_t ress)
{
LZ4F_errorCode_t errorCode = LZ4F_freeDecompressionContext(ress.dCtx);
- if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode));
+ if (LZ4F_isError(errorCode)) END_PROCESS(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode));
free(ress.srcBuffer);
free(ress.dstBuffer);
free(ress.dictBuffer);
@@ -1066,7 +1078,7 @@ LZ4IO_decompressLZ4F(dRess_t ress,
size_t outSize= 0;
LZ4IO_writeLE32(ress.srcBuffer, LZ4IO_MAGICNUMBER);
nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, ress.dstBuffer, &outSize, ress.srcBuffer, &inSize, ress.dictBuffer, ress.dictBufferSize, NULL);
- if (LZ4F_isError(nextToLoad)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(nextToLoad));
+ if (LZ4F_isError(nextToLoad)) END_PROCESS(62, "Header error : %s", LZ4F_getErrorName(nextToLoad));
}
/* Main Loop */
@@ -1085,7 +1097,7 @@ LZ4IO_decompressLZ4F(dRess_t ress,
size_t remaining = readSize - pos;
decodedBytes = ress.dstBufferSize;
nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, ress.dictBuffer, ress.dictBufferSize, NULL);
- if (LZ4F_isError(nextToLoad)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad));
+ if (LZ4F_isError(nextToLoad)) END_PROCESS(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad));
pos += remaining;
/* Write Block */
@@ -1100,10 +1112,10 @@ LZ4IO_decompressLZ4F(dRess_t ress,
}
}
/* can be out because readSize == 0, which could be an fread() error */
- if (ferror(srcFile)) EXM_THROW(67, "Read error");
+ if (ferror(srcFile)) END_PROCESS(67, "Read error");
if (!prefs->testMode) LZ4IO_fwriteSparseEnd(dstFile, storedSkips);
- if (nextToLoad!=0) EXM_THROW(68, "Unfinished stream");
+ if (nextToLoad!=0) END_PROCESS(68, "Unfinished stream");
return filesize;
}
@@ -1127,14 +1139,14 @@ LZ4IO_passThrough(FILE* finput, FILE* foutput,
unsigned storedSkips = 0;
if (fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput) != MAGICNUMBER_SIZE) {
- EXM_THROW(50, "Pass-through write error");
+ END_PROCESS(50, "Pass-through write error");
}
while (readBytes) {
readBytes = fread(buffer, 1, sizeof(buffer), finput);
total += readBytes;
storedSkips = LZ4IO_fwriteSparse(foutput, buffer, readBytes, sparseFileSupport, storedSkips);
}
- if (ferror(finput)) EXM_THROW(51, "Read Error");
+ if (ferror(finput)) END_PROCESS(51, "Read Error");
LZ4IO_fwriteSparseEnd(foutput, storedSkips);
return total;
@@ -1180,7 +1192,7 @@ selectDecoder(dRess_t ress,
size_t const nbReadBytes = fread(MNstore, 1, MAGICNUMBER_SIZE, finput);
if (nbReadBytes==0) { nbFrames = 0; return ENDOFSTREAM; } /* EOF */
if (nbReadBytes != MAGICNUMBER_SIZE)
- EXM_THROW(40, "Unrecognized header : Magic Number unreadable");
+ END_PROCESS(40, "Unrecognized header : Magic Number unreadable");
magicNumber = LZ4IO_readLE32(MNstore); /* Little Endian format */
}
if (LZ4IO_isSkippableMagicNumber(magicNumber))
@@ -1197,12 +1209,12 @@ selectDecoder(dRess_t ress,
DISPLAYLEVEL(4, "Skipping detected skippable area \n");
{ size_t const nbReadBytes = fread(MNstore, 1, 4, finput);
if (nbReadBytes != 4)
- EXM_THROW(42, "Stream error : skippable size unreadable");
+ END_PROCESS(42, "Stream error : skippable size unreadable");
}
{ unsigned const size = LZ4IO_readLE32(MNstore);
int const errorNb = fseek_u32(finput, size, SEEK_CUR);
if (errorNb != 0)
- EXM_THROW(43, "Stream error : cannot skip skippable area");
+ END_PROCESS(43, "Stream error : cannot skip skippable area");
}
return 0;
default:
@@ -1212,7 +1224,7 @@ selectDecoder(dRess_t ress,
nbFrames = 0;
return LZ4IO_passThrough(finput, foutput, MNstore, prefs->sparseFileSupport);
}
- EXM_THROW(44,"Unrecognized header : file cannot be decoded");
+ END_PROCESS(44,"Unrecognized header : file cannot be decoded");
}
{ long int const position = ftell(finput); /* only works for files < 2 GB */
DISPLAYLEVEL(2, "Stream followed by undecodable data ");
@@ -1252,7 +1264,7 @@ LZ4IO_decompressSrcFile(dRess_t ress,
fclose(finput);
if (prefs->removeSrcFile) { /* --rm */
if (remove(input_filename))
- EXM_THROW(45, "Remove error : %s: %s", input_filename, strerror(errno));
+ END_PROCESS(45, "Remove error : %s: %s", input_filename, strerror(errno));
}
/* Final Status */
@@ -1275,7 +1287,7 @@ LZ4IO_decompressDstFile(dRess_t ress,
FILE* const foutput = LZ4IO_openDstFile(output_filename, prefs);
if (foutput==NULL) return 1; /* failure */
- if ( strcmp(input_filename, stdinmark)
+ if ( !LZ4IO_isStdin(input_filename)
&& UTIL_getFileStat(input_filename, &statbuf))
stat_result = 1;
@@ -1286,8 +1298,8 @@ LZ4IO_decompressDstFile(dRess_t ress,
/* Copy owner, file permissions and modification time */
if ( stat_result != 0
- && strcmp (output_filename, stdoutmark)
- && strcmp (output_filename, nulmark)) {
+ && !LZ4IO_isStdout(output_filename)
+ && !LZ4IO_isDevNull(output_filename)) {
UTIL_setFileStat(output_filename, &statbuf);
/* should return value be read ? or is silent fail good enough ? */
}
@@ -1325,23 +1337,23 @@ int LZ4IO_decompressMultipleFilenames(
size_t const suffixSize = strlen(suffix);
dRess_t ress = LZ4IO_createDResources(prefs);
- if (outFileName==NULL) EXM_THROW(70, "Memory allocation error");
+ if (outFileName==NULL) END_PROCESS(70, "Memory allocation error");
ress.dstFile = LZ4IO_openDstFile(stdoutmark, prefs);
for (i=0; i<ifntSize; i++) {
size_t const ifnSize = strlen(inFileNamesTable[i]);
const char* const suffixPtr = inFileNamesTable[i] + ifnSize - suffixSize;
- if (!strcmp(suffix, stdoutmark)) {
- missingFiles += LZ4IO_decompressSrcFile(ress, inFileNamesTable[i], stdoutmark, prefs);
+ if (LZ4IO_isStdout(suffix) || LZ4IO_isDevNull(suffix)) {
+ missingFiles += LZ4IO_decompressSrcFile(ress, inFileNamesTable[i], suffix, prefs);
continue;
}
if (ofnSize <= ifnSize-suffixSize+1) {
free(outFileName);
ofnSize = ifnSize + 20;
outFileName = (char*)malloc(ofnSize);
- if (outFileName==NULL) EXM_THROW(71, "Memory allocation error");
+ if (outFileName==NULL) END_PROCESS(71, "Memory allocation error");
}
- if (ifnSize <= suffixSize || strcmp(suffixPtr, suffix) != 0) {
+ if (ifnSize <= suffixSize || !UTIL_sameString(suffixPtr, suffix) ) {
DISPLAYLEVEL(1, "File extension doesn't match expected LZ4_EXTENSION (%4s); will not process file: %s\n", suffix, inFileNamesTable[i]);
skippedFiles++;
continue;
@@ -1459,7 +1471,7 @@ static unsigned long long LZ4IO_skipLegacyBlocksData(FILE* finput)
|| LZ4IO_isSkippableMagicNumber(nextCBlockSize) ) {
/* Rewind back. we want cursor at the beginning of next frame */
if (UTIL_fseek(finput, -LZ4IO_LEGACY_BLOCK_HEADER_SIZE, SEEK_CUR) != 0) {
- EXM_THROW(37, "impossible to skip backward");
+ END_PROCESS(37, "impossible to skip backward");
}
break;
}
@@ -1531,7 +1543,7 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam
if (nbReadBytes == 0) { break; } /* EOF */
result = LZ4IO_format_not_known; /* default result (error) */
if (nbReadBytes != MAGICNUMBER_SIZE) {
- EXM_THROW(40, "Unrecognized header : Magic Number unreadable");
+ END_PROCESS(40, "Unrecognized header : Magic Number unreadable");
} }
magicNumber = LZ4IO_readLE32(buffer); /* Little Endian format */
if (LZ4IO_isSkippableMagicNumber(magicNumber))
@@ -1542,14 +1554,14 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam
if (cfinfo->frameSummary.frameType != lz4Frame) cfinfo->eqFrameTypes = 0;
/* Get frame info */
{ const size_t readBytes = fread(buffer + MAGICNUMBER_SIZE, 1, LZ4F_HEADER_SIZE_MIN - MAGICNUMBER_SIZE, finput);
- if (!readBytes || ferror(finput)) EXM_THROW(71, "Error reading %s", input_filename);
+ if (!readBytes || ferror(finput)) END_PROCESS(71, "Error reading %s", input_filename);
}
{ size_t hSize = LZ4F_headerSize(&buffer, LZ4F_HEADER_SIZE_MIN);
if (LZ4F_isError(hSize)) break;
if (hSize > (LZ4F_HEADER_SIZE_MIN + MAGICNUMBER_SIZE)) {
/* We've already read LZ4F_HEADER_SIZE_MIN so read any extra until hSize*/
const size_t readBytes = fread(buffer + LZ4F_HEADER_SIZE_MIN, 1, hSize - LZ4F_HEADER_SIZE_MIN, finput);
- if (!readBytes || ferror(finput)) EXM_THROW(72, "Error reading %s", input_filename);
+ if (!readBytes || ferror(finput)) END_PROCESS(72, "Error reading %s", input_filename);
}
/* Create decompression context */
{ LZ4F_dctx* dctx;
@@ -1617,12 +1629,12 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam
cfinfo->allContentSize = 0;
{ size_t const nbReadBytes = fread(buffer, 1, 4, finput);
if (nbReadBytes != 4)
- EXM_THROW(42, "Stream error : skippable size unreadable");
+ END_PROCESS(42, "Stream error : skippable size unreadable");
}
{ unsigned const size = LZ4IO_readLE32(buffer);
int const errorNb = fseek_u32(finput, size, SEEK_CUR);
if (errorNb != 0)
- EXM_THROW(43, "Stream error : cannot skip skippable area");
+ END_PROCESS(43, "Stream error : cannot skip skippable area");
DISPLAYLEVEL(3, " %6llu %14s %5s %8s %20u %20s %9s\n",
cfinfo->frameCount + 1,
"SkippableFrame",
@@ -1662,7 +1674,7 @@ int LZ4IO_displayCompressedFilesInfo(const char** inFileNames, size_t ifnIdx)
/* Get file info */
LZ4IO_cFileInfo_t cfinfo = LZ4IO_INIT_CFILEINFO;
cfinfo.fileName = LZ4IO_baseName(inFileNames[idx]);
- if ((strcmp(inFileNames[idx], stdinmark) == 0) ? !UTIL_isRegFD(0) : !UTIL_isRegFile(inFileNames[idx])) {
+ if (LZ4IO_isStdin(inFileNames[idx]) ? !UTIL_isRegFD(0) : !UTIL_isRegFile(inFileNames[idx])) {
DISPLAYLEVEL(1, "lz4: %s is not a regular file \n", inFileNames[idx]);
return 1;
}
diff --git a/programs/util.h b/programs/util.h
index 2840179..3192ddc 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -175,6 +175,39 @@ extern "C" {
#endif
+
+/*-****************************************
+* Allocation functions
+******************************************/
+/*
+ * A modified version of realloc().
+ * If UTIL_realloc() fails the original block is freed.
+*/
+UTIL_STATIC void* UTIL_realloc(void* ptr, size_t size)
+{
+ void* const newptr = realloc(ptr, size);
+ if (newptr) return newptr;
+ free(ptr);
+ return NULL;
+}
+
+
+/*-****************************************
+* String functions
+******************************************/
+/*
+ * A modified version of realloc().
+ * If UTIL_realloc() fails the original block is freed.
+*/
+UTIL_STATIC int UTIL_sameString(const char* a, const char* b)
+{
+ assert(a!=NULL && b!=NULL); /* unsupported scenario */
+ if (a==NULL) return 0;
+ if (b==NULL) return 0;
+ return !strcmp(a,b);
+}
+
+
/*-****************************************
* Time functions
******************************************/
@@ -451,19 +484,6 @@ UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFi
}
-/*
- * A modified version of realloc().
- * If UTIL_realloc() fails the original block is freed.
-*/
-UTIL_STATIC void* UTIL_realloc(void* ptr, size_t size)
-{
- void* const newptr = realloc(ptr, size);
- if (newptr) return newptr;
- free(ptr);
- return NULL;
-}
-
-
#ifdef _WIN32
# define UTIL_HAS_CREATEFILELIST
@@ -665,8 +685,8 @@ UTIL_createFileList(const char** inputNames, unsigned inputNamesNb,
UTIL_STATIC void
UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer)
{
- if (allocatedBuffer) free(allocatedBuffer);
- if (filenameTable) free((void*)filenameTable);
+ free(allocatedBuffer);
+ free((void*)filenameTable);
}
diff --git a/tests/.gitignore b/tests/.gitignore
index 99351af..346c989 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -12,7 +12,7 @@ roundTripTest
checkTag
checkFrame
decompress-partial
-
+decompress-partial-usingDict
# test artefacts
tmp*
versionsTest
diff --git a/tests/Makefile b/tests/Makefile
index b4d40ca..b67f135 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -112,6 +112,9 @@ checkFrame : lz4frame.o lz4.o lz4hc.o xxhash.o checkFrame.c
decompress-partial: lz4.o decompress-partial.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
+decompress-partial-usingDict: lz4.o decompress-partial-usingDict.c
+ $(CC) $(FLAGS) $^ -o $@$(EXT)
+
.PHONY: clean
clean:
@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
@@ -151,6 +154,7 @@ endif
# note : we should probably settle on a single compare utility
CMP:=cmp
+GREP:=grep
DIFF:=diff
ifneq (,$(filter $(shell $(UNAME)),SunOS))
DIFF:=gdiff
@@ -286,6 +290,11 @@ test-lz4-multiple: lz4 datagen
$(CMP) tmp-tlm-concat1 tmp-tlm-concat2 # must be equivalent
# compress multiple files, one of which is absent (must fail)
! $(LZ4) -f -m tmp-tlm-concat1 notHere tmp-tlm-concat2 # must fail : notHere not present
+ # test lz4-compressed file
+ $(LZ4) -tm tmp-tlm-concat1.lz4
+ $(LZ4) -tm tmp-tlm-concat1.lz4 tmp-tlm-concat2.lz4
+ # test multiple lz4 files, one of which is absent (must fail)
+ ! $(LZ4) -tm tmp-tlm-concat1.lz4 notHere.lz4 tmp-tlm-concat2.lz4
@$(RM) tmp-tlm*
test-lz4-multiple-legacy: lz4 datagen
@@ -344,8 +353,7 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat
$(LZ4) --rm -f tmp-tlb-hw tmp-tlb-hw.lz4
test ! -f tmp-tlb-hw # must fail (--rm)
test -f tmp-tlb-hw.lz4
- $(PRGDIR)/lz4cat tmp-tlb-hw.lz4 # must display hello world
- test -f tmp-tlb-hw.lz4
+ $(PRGDIR)/lz4cat tmp-tlb-hw.lz4 | $(GREP) "hello world"
$(PRGDIR)/unlz4 --rm tmp-tlb-hw.lz4 tmp-tlb-hw
test -f tmp-tlb-hw
test ! -f tmp-tlb-hw.lz4 # must fail (--rm)
@@ -366,7 +374,8 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat
$(LZ4) -d --rm -- -z tmp-tlb4 # uncompresses ./-z into tmp-tlb4
test ! -f ./-z
$(DIFF) -q tmp-tlb-hw tmp-tlb4
- $(LZ4) -f tmp-tlb-hw
+ ! $(LZ4) tmp-tlb2 tmp-tlb3 tmp-tlb4 # must fail: refuse to handle 3+ file names
+ $(LZ4) -f tmp-tlb-hw # create tmp-tlb-hw.lz4, for next tests
$(LZ4) --list tmp-tlb-hw.lz4 # test --list on valid single-frame file
$(LZ4) --list < tmp-tlb-hw.lz4 # test --list from stdin (file only)
$(CAT) tmp-tlb-hw >> tmp-tlb-hw.lz4
@@ -389,7 +398,6 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat
@$(RM) tmp-tlb*
-
test-lz4-dict: lz4 datagen
@echo "\n ---- test lz4 compression/decompression with dictionary ----"
$(DATAGEN) -g16KB > tmp-dict
@@ -412,7 +420,6 @@ test-lz4-dict: lz4 datagen
< tmp-dict-$$l $(LZ4) -D stdin tmp-dict-data-128KB -c | $(LZ4) -dD tmp-dict-$$l-tail | $(DIFF) - tmp-dict-data-128KB; \
< tmp-dict-$$l-tail $(LZ4) -D stdin tmp-dict-data-128KB -c | $(LZ4) -dD tmp-dict-$$l | $(DIFF) - tmp-dict-data-128KB; \
done
-
@$(RM) tmp-dict*
test-lz4-hugefile: lz4 datagen
@@ -547,8 +554,10 @@ test-mem: lz4 datagen fuzzer frametest fullbench
test-mem32: lz4c32 datagen
# unfortunately, valgrind doesn't seem to work with non-native binary...
-test-decompress-partial : decompress-partial
+test-decompress-partial : decompress-partial decompress-partial-usingDict
@echo "\n ---- test decompress-partial ----"
./decompress-partial$(EXT)
+ @echo "\n ---- test decompress-partial-usingDict ----"
+ ./decompress-partial-usingDict$(EXT)
endif
diff --git a/tests/decompress-partial-usingDict.c b/tests/decompress-partial-usingDict.c
new file mode 100644
index 0000000..cfcb971
--- /dev/null
+++ b/tests/decompress-partial-usingDict.c
@@ -0,0 +1,88 @@
+#include "stdio.h"
+#include "string.h"
+#include "stdlib.h"
+#include "lz4.h"
+
+const char source[] =
+ "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod\n"
+ "tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim\n"
+ "veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea\n"
+ "commodo consequat. Duis aute irure dolor in reprehenderit in voluptate\n"
+ "velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat\n"
+ "cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id\n"
+ "est laborum.\n"
+ "\n"
+ "Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium\n"
+ "doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore\n"
+ "veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim\n"
+ "ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia\n"
+ "consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque\n"
+ "porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur,\n"
+ "adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore\n"
+ "et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis\n"
+ "nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid\n"
+ "ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea\n"
+ "voluptate velit esse quam nihil molestiae consequatur, vel illum qui\n"
+ "dolorem eum fugiat quo voluptas nulla pariatur?\n";
+
+#define BUFFER_SIZE 2048
+
+int main(void)
+{
+ int srcLen = (int)strlen(source);
+ size_t const smallSize = 1024;
+ size_t const largeSize = 64 * 1024 - 1;
+ char cmpBuffer[BUFFER_SIZE];
+ char buffer[BUFFER_SIZE + largeSize];
+ char* outBuffer = buffer + largeSize;
+ char* const dict = (char*)malloc(largeSize);
+ char* const largeDict = dict;
+ char* const smallDict = dict + largeSize - smallSize;
+ int cmpSize;
+ int i;
+
+ cmpSize = LZ4_compress_default(source, cmpBuffer, srcLen, BUFFER_SIZE);
+
+ for (i = cmpSize; i < cmpSize + 10; ++i) {
+ int result = LZ4_decompress_safe_partial_usingDict(cmpBuffer, outBuffer, i, srcLen, BUFFER_SIZE, NULL, 0);
+ if ((result < 0) || (result != srcLen) || memcmp(source, outBuffer, srcLen)) {
+ printf("test decompress-partial-usingDict with no dict error \n");
+ return -1;
+ }
+ }
+
+ for (i = cmpSize; i < cmpSize + 10; ++i) {
+ int result = LZ4_decompress_safe_partial_usingDict(cmpBuffer, outBuffer, i, srcLen, BUFFER_SIZE, outBuffer - smallSize, smallSize);
+ if ((result < 0) || (result != srcLen) || memcmp(source, outBuffer, srcLen)) {
+ printf("test decompress-partial-usingDict with small prefix error \n");
+ return -1;
+ }
+ }
+
+ for (i = cmpSize; i < cmpSize + 10; ++i) {
+ int result = LZ4_decompress_safe_partial_usingDict(cmpBuffer, outBuffer, i, srcLen, BUFFER_SIZE, buffer, largeSize);
+ if ((result < 0) || (result != srcLen) || memcmp(source, outBuffer, srcLen)) {
+ printf("test decompress-partial-usingDict with large prefix error \n");
+ return -1;
+ }
+ }
+
+ for (i = cmpSize; i < cmpSize + 10; ++i) {
+ int result = LZ4_decompress_safe_partial_usingDict(cmpBuffer, outBuffer, i, srcLen, BUFFER_SIZE, smallDict, smallSize);
+ if ((result < 0) || (result != srcLen) || memcmp(source, outBuffer, srcLen)) {
+ printf("test decompress-partial-usingDict with small external dict error \n");
+ return -1;
+ }
+ }
+
+ for (i = cmpSize; i < cmpSize + 10; ++i) {
+ int result = LZ4_decompress_safe_partial_usingDict(cmpBuffer, outBuffer, i, srcLen, BUFFER_SIZE, largeDict, largeSize);
+ if ((result < 0) || (result != srcLen) || memcmp(source, outBuffer, srcLen)) {
+ printf("test decompress-partial-usingDict with large external dict error \n");
+ return -1;
+ }
+ }
+
+ printf("test decompress-partial-usingDict OK \n");
+ return 0;
+}
diff --git a/tests/fullbench.c b/tests/fullbench.c
index 55bf0b7..ec20dcb 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -312,6 +312,13 @@ static int local_LZ4_decompress_safe_usingDict(const char* in, char* out, int in
return outSize;
}
+static int local_LZ4_decompress_safe_partial_usingDict(const char* in, char* out, int inSize, int outSize)
+{
+ int result = LZ4_decompress_safe_partial_usingDict(in, out, inSize, outSize - 5, outSize, out - 65536, 65536);
+ if (result < 0) return result;
+ return outSize;
+}
+
#ifndef LZ4_DLL_IMPORT
#if defined (__cplusplus)
extern "C" {
@@ -325,12 +332,30 @@ extern int LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSiz
static int local_LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize)
{
- (void)inSize;
LZ4_decompress_safe_forceExtDict(in, out, inSize, outSize, out - 65536, 65536);
return outSize;
}
#endif
+#ifndef LZ4_DLL_IMPORT
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+extern int LZ4_decompress_safe_partial_forceExtDict(const char* in, char* out, int inSize, int targetOutputSize, int dstCapacity, const void* dict, size_t dictSize);
+
+#if defined (__cplusplus)
+}
+#endif
+
+static int local_LZ4_decompress_safe_partial_forceExtDict(const char* in, char* out, int inSize, int outSize)
+{
+ int result = LZ4_decompress_safe_partial_forceExtDict(in, out, inSize, outSize - 5, outSize, out - 65536, 65536);
+ if (result < 0) return result;
+ return outSize;
+}
+#endif
+
static int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSize, int outSize)
{
int result = LZ4_decompress_safe_partial(in, out, inSize, outSize - 5, outSize);
@@ -657,15 +682,17 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
case 5: decompressionFunction = local_LZ4_decompress_safe_withPrefix64k; dName = "LZ4_decompress_safe_withPrefix64k"; break;
case 6: decompressionFunction = local_LZ4_decompress_safe_usingDict; dName = "LZ4_decompress_safe_usingDict"; break;
case 7: decompressionFunction = local_LZ4_decompress_safe_partial; dName = "LZ4_decompress_safe_partial"; checkResult = 0; break;
+ case 8: decompressionFunction = local_LZ4_decompress_safe_partial_usingDict; dName = "LZ4_decompress_safe_partial_usingDict"; checkResult = 0; break;
#ifndef LZ4_DLL_IMPORT
- case 8: decompressionFunction = local_LZ4_decompress_safe_forceExtDict; dName = "LZ4_decompress_safe_forceExtDict"; break;
+ case 9: decompressionFunction = local_LZ4_decompress_safe_partial_forceExtDict; dName = "LZ4_decompress_safe_partial_forceExtDict"; checkResult = 0; break;
+ case 10: decompressionFunction = local_LZ4_decompress_safe_forceExtDict; dName = "LZ4_decompress_safe_forceExtDict"; break;
#endif
- case 10:
case 11:
case 12:
- if (dAlgNb == 10) { decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress"; } /* can be skipped */
- if (dAlgNb == 11) { decompressionFunction = local_LZ4F_decompress_followHint; dName = "LZ4F_decompress_followHint"; } /* can be skipped */
- if (dAlgNb == 12) { decompressionFunction = local_LZ4F_decompress_noHint; dName = "LZ4F_decompress_noHint"; } /* can be skipped */
+ case 13:
+ if (dAlgNb == 11) { decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress"; } /* can be skipped */
+ if (dAlgNb == 12) { decompressionFunction = local_LZ4F_decompress_followHint; dName = "LZ4F_decompress_followHint"; } /* can be skipped */
+ if (dAlgNb == 13) { decompressionFunction = local_LZ4F_decompress_noHint; dName = "LZ4F_decompress_noHint"; } /* can be skipped */
/* prepare compressed data using frame format */
{ size_t const fcsize = LZ4F_compressFrame(compressed_buff, (size_t)compressedBuffSize, orig_buff, benchedSize, NULL);
assert(!LZ4F_isError(fcsize));
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index e6fa13c..07d63a2 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -630,6 +630,46 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(memcmp(block, decodedBuffer, (size_t)targetSize), "LZ4_decompress_safe_partial: corruption detected in regenerated data");
}
+ /* Partial decompression using dictionary. */
+ FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial_usingDict using no dict");
+ { size_t const missingOutBytes = FUZ_rand(&randState) % (unsigned)blockSize;
+ int const targetSize = (int)((size_t)blockSize - missingOutBytes);
+ size_t const extraneousInBytes = FUZ_rand(&randState) % 2;
+ int const inCSize = (int)((size_t)compressedSize + extraneousInBytes);
+ char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A;
+ int const decResult = LZ4_decompress_safe_partial_usingDict(compressedBuffer, decodedBuffer, inCSize, targetSize, blockSize, NULL, 0);
+ FUZ_CHECKTEST(decResult<0, "LZ4_decompress_safe_partial_usingDict failed despite valid input data (error:%i)", decResult);
+ FUZ_CHECKTEST(decResult != targetSize, "LZ4_decompress_safe_partial_usingDict did not regenerated required amount of data (%i < %i <= %i)", decResult, targetSize, blockSize);
+ FUZ_CHECKTEST(decodedBuffer[targetSize] != sentinel, "LZ4_decompress_safe_partial_usingDict overwrite beyond requested size (though %i <= %i <= %i)", decResult, targetSize, blockSize);
+ FUZ_CHECKTEST(memcmp(block, decodedBuffer, (size_t)targetSize), "LZ4_decompress_safe_partial_usingDict: corruption detected in regenerated data");
+ }
+
+ FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial_usingDict() using prefix as dict");
+ { size_t const missingOutBytes = FUZ_rand(&randState) % (unsigned)blockSize;
+ int const targetSize = (int)((size_t)blockSize - missingOutBytes);
+ size_t const extraneousInBytes = FUZ_rand(&randState) % 2;
+ int const inCSize = (int)((size_t)compressedSize + extraneousInBytes);
+ char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A;
+ int const decResult = LZ4_decompress_safe_partial_usingDict(compressedBuffer, decodedBuffer, inCSize, targetSize, blockSize, decodedBuffer, dictSize);
+ FUZ_CHECKTEST(decResult<0, "LZ4_decompress_safe_partial_usingDict failed despite valid input data (error:%i)", decResult);
+ FUZ_CHECKTEST(decResult != targetSize, "LZ4_decompress_safe_partial_usingDict did not regenerated required amount of data (%i < %i <= %i)", decResult, targetSize, blockSize);
+ FUZ_CHECKTEST(decodedBuffer[targetSize] != sentinel, "LZ4_decompress_safe_partial_usingDict overwrite beyond requested size (though %i <= %i <= %i)", decResult, targetSize, blockSize);
+ FUZ_CHECKTEST(memcmp(block, decodedBuffer, (size_t)targetSize), "LZ4_decompress_safe_partial_usingDict: corruption detected in regenerated data");
+ }
+
+ FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial_usingDict() using external dict");
+ { size_t const missingOutBytes = FUZ_rand(&randState) % (unsigned)blockSize;
+ int const targetSize = (int)((size_t)blockSize - missingOutBytes);
+ size_t const extraneousInBytes = FUZ_rand(&randState) % 2;
+ int const inCSize = (int)((size_t)compressedSize + extraneousInBytes);
+ char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A;
+ int const decResult = LZ4_decompress_safe_partial_usingDict(compressedBuffer, decodedBuffer, inCSize, targetSize, blockSize, dict, dictSize);
+ FUZ_CHECKTEST(decResult<0, "LZ4_decompress_safe_partial_usingDict failed despite valid input data (error:%i)", decResult);
+ FUZ_CHECKTEST(decResult != targetSize, "LZ4_decompress_safe_partial_usingDict did not regenerated required amount of data (%i < %i <= %i)", decResult, targetSize, blockSize);
+ FUZ_CHECKTEST(decodedBuffer[targetSize] != sentinel, "LZ4_decompress_safe_partial_usingDict overwrite beyond requested size (though %i <= %i <= %i)", decResult, targetSize, blockSize);
+ FUZ_CHECKTEST(memcmp(block, decodedBuffer, (size_t)targetSize), "LZ4_decompress_safe_partial_usingDict: corruption detected in regenerated data");
+ }
+
/* Test Compression with limited output size */
/* Test compression with output size being exactly what's necessary (should work) */
generated by cgit v0.12 at 2024-09-18 18:34:24 (GMT)