diff options
-rw-r--r-- | lib/lz4frame.c | 15 | ||||
-rw-r--r-- | lib/lz4frame.h | 1 | ||||
-rw-r--r-- | lib/lz4hc.c | 20 | ||||
-rw-r--r-- | programs/lz4cli.c | 93 | ||||
-rw-r--r-- | programs/lz4io.c | 269 | ||||
-rw-r--r-- | programs/lz4io.h | 44 | ||||
-rw-r--r-- | tests/.gitignore | 1 | ||||
-rw-r--r-- | tests/Makefile | 14 | ||||
-rw-r--r-- | tests/frametest.c | 23 | ||||
-rw-r--r-- | tests/fuzzer.c | 118 |
10 files changed, 373 insertions, 225 deletions
diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 705832d..3f81ef0 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -265,22 +265,21 @@ unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; } int LZ4F_compressionLevel_max(void) { return LZ4HC_CLEVEL_MAX; } - -/*-************************************ -* Private functions -**************************************/ -#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) - -static size_t LZ4F_getBlockSize(unsigned blockSizeID) +size_t LZ4F_getBlockSize(unsigned blockSizeID) { static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB }; if (blockSizeID == 0) blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; + if (blockSizeID < 4 || blockSizeID > 7) return err0r(LZ4F_ERROR_maxBlockSize_invalid); blockSizeID -= 4; - if (blockSizeID > 3) return err0r(LZ4F_ERROR_maxBlockSize_invalid); return blockSizes[blockSizeID]; } +/*-************************************ +* Private functions +**************************************/ +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + static BYTE LZ4F_headerChecksum (const void* header, size_t length) { U32 const xxh = XXH32(header, length, 0); diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 7c7c34e..68f4118 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -483,6 +483,7 @@ typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult); +LZ4FLIB_STATIC_API size_t LZ4F_getBlockSize(unsigned); /********************************** * Bulk processing dictionary API diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 56c8f47..129bf0c 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -88,6 +88,8 @@ typedef enum { #define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG)) #define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */ #define DELTANEXTU16(table, pos) table[(U16)(pos)] /* faster */ +/* Make fields passed to, and updated by LZ4HC_encodeSequence explicit */ +#define UPDATABLE(ip, op, anchor) &ip, &op, &anchor static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } @@ -437,7 +439,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( /* Encode Literal length */ length = (size_t)(*ip - *anchor); - if ((limit) && ((*op + (length >> 8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */ + if ((limit) && ((*op + (length / 255) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */ if (length >= RUN_MASK) { size_t len = length - RUN_MASK; *token = (RUN_MASK << ML_BITS); @@ -458,7 +460,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( /* Encode MatchLength */ assert(matchLength >= MINMATCH); length = (size_t)(matchLength - MINMATCH); - if ((limit) && (*op + (length >> 8) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */ + if ((limit) && (*op + (length / 255) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */ if (length >= ML_MASK) { *token += ML_MASK; length -= ML_MASK; @@ -533,7 +535,7 @@ _Search2: if (ml2 == ml) { /* No better match => encode ML1 */ optr = op; - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow; continue; } @@ -581,10 +583,10 @@ _Search3: if (start2 < ip+ml) ml = (int)(start2 - ip); /* Now, encode 2 sequences */ optr = op; - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow; ip = start2; optr = op; - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) goto _dest_overflow; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml2, ref2, limit, oend)) goto _dest_overflow; continue; } @@ -603,7 +605,7 @@ _Search3: } optr = op; - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow; ip = start3; ref = ref3; ml = ml3; @@ -641,7 +643,7 @@ _Search3: } } optr = op; - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow; /* ML2 becomes ML1 */ ip = start2; ref = ref2; ml = ml2; @@ -1206,7 +1208,7 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, int const firstML = firstMatch.len; const BYTE* const matchPos = ip - firstMatch.off; opSaved = op; - if ( LZ4HC_encodeSequence(&ip, &op, &anchor, firstML, matchPos, limit, oend) ) /* updates ip, op and anchor */ + if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), firstML, matchPos, limit, oend) ) /* updates ip, op and anchor */ goto _dest_overflow; continue; } @@ -1378,7 +1380,7 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, assert(ml >= MINMATCH); assert((offset >= 1) && (offset <= MAX_DISTANCE)); opSaved = op; - if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) ) /* updates ip, op and anchor */ + if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ip - offset, limit, oend) ) /* updates ip, op and anchor */ goto _dest_overflow; } } } /* while (ip <= mflimit) */ diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 3709f50..464e43b 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -92,7 +92,7 @@ static unsigned displayLevel = 2; /* 0 : no display ; 1: errors only ; 2 : dow ***************************************/ #define DEFAULT_COMPRESSOR LZ4IO_compressFilename #define DEFAULT_DECOMPRESSOR LZ4IO_decompressFilename -int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel); /* hidden function */ +int LZ4IO_compressFilename_Legacy(LZ4IO_prefs_t* const prefs, const char* input_filename, const char* output_filename, int compressionlevel); /* hidden function */ /*-*************************** @@ -287,6 +287,19 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) typedef enum { om_auto, om_compress, om_decompress, om_test, om_bench } operationMode_e; +/** determineOpMode() : + * auto-determine operation mode, based on input filename extension + * @return `om_decompress` if input filename has .lz4 extension and `om_compress` otherwise. + */ +static operationMode_e determineOpMode(const char* inputFilename) +{ + size_t const inSize = strlen(inputFilename); + size_t const extSize = strlen(LZ4_EXTENSION); + size_t const extStart= (inSize > extSize) ? inSize-extSize : 0; + if (!strcmp(inputFilename+extStart, LZ4_EXTENSION)) return om_decompress; + else return om_compress; +} + int main(int argc, const char** argv) { int i, @@ -305,9 +318,10 @@ int main(int argc, const char** argv) char* dynNameSpace = NULL; const char** inFileNames = (const char**) calloc(argc, sizeof(char*)); unsigned ifnIdx=0; + LZ4IO_prefs_t* const prefs = LZ4IO_defaultPreferences(); const char nullOutput[] = NULL_OUTPUT; const char extension[] = LZ4_EXTENSION; - size_t blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT); + size_t blockSize = LZ4IO_setBlockSizeID(prefs, LZ4_BLOCKSIZEID_DEFAULT); const char* const exeName = lastNameFromPath(argv[0]); #ifdef UTIL_HAS_CREATEFILELIST const char** extendedFileList = NULL; @@ -321,13 +335,14 @@ int main(int argc, const char** argv) return 1; } inFileNames[0] = stdinmark; - LZ4IO_setOverwrite(0); + LZ4IO_setOverwrite(prefs, 0); /* predefined behaviors, based on binary/link name */ if (exeNameMatch(exeName, LZ4CAT)) { mode = om_decompress; - LZ4IO_setOverwrite(1); - LZ4IO_setRemoveSrcFile(0); + LZ4IO_setOverwrite(prefs, 1); + LZ4IO_setPassThrough(prefs, 1); + LZ4IO_setRemoveSrcFile(prefs, 0); forceStdout=1; output_filename=stdoutmark; displayLevel=1; @@ -359,23 +374,23 @@ int main(int argc, const char** argv) || (!strcmp(argument, "--uncompress"))) { mode = om_decompress; continue; } if (!strcmp(argument, "--multiple")) { multiple_inputs = 1; continue; } if (!strcmp(argument, "--test")) { mode = om_test; continue; } - if (!strcmp(argument, "--force")) { LZ4IO_setOverwrite(1); continue; } - if (!strcmp(argument, "--no-force")) { LZ4IO_setOverwrite(0); continue; } + if (!strcmp(argument, "--force")) { LZ4IO_setOverwrite(prefs, 1); continue; } + if (!strcmp(argument, "--no-force")) { LZ4IO_setOverwrite(prefs, 0); continue; } if ((!strcmp(argument, "--stdout")) || (!strcmp(argument, "--to-stdout"))) { forceStdout=1; output_filename=stdoutmark; continue; } - if (!strcmp(argument, "--frame-crc")) { LZ4IO_setStreamChecksumMode(1); continue; } - if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(0); continue; } - if (!strcmp(argument, "--content-size")) { LZ4IO_setContentSize(1); continue; } - if (!strcmp(argument, "--no-content-size")) { LZ4IO_setContentSize(0); continue; } - if (!strcmp(argument, "--sparse")) { LZ4IO_setSparseFile(2); continue; } - if (!strcmp(argument, "--no-sparse")) { LZ4IO_setSparseFile(0); continue; } - if (!strcmp(argument, "--favor-decSpeed")) { LZ4IO_favorDecSpeed(1); continue; } + if (!strcmp(argument, "--frame-crc")) { LZ4IO_setStreamChecksumMode(prefs, 1); continue; } + if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(prefs, 0); continue; } + if (!strcmp(argument, "--content-size")) { LZ4IO_setContentSize(prefs, 1); continue; } + if (!strcmp(argument, "--no-content-size")) { LZ4IO_setContentSize(prefs, 0); continue; } + if (!strcmp(argument, "--sparse")) { LZ4IO_setSparseFile(prefs, 2); continue; } + if (!strcmp(argument, "--no-sparse")) { LZ4IO_setSparseFile(prefs, 0); continue; } + if (!strcmp(argument, "--favor-decSpeed")) { LZ4IO_favorDecSpeed(prefs, 1); continue; } if (!strcmp(argument, "--verbose")) { displayLevel++; continue; } if (!strcmp(argument, "--quiet")) { if (displayLevel) displayLevel--; continue; } if (!strcmp(argument, "--version")) { DISPLAY(WELCOME_MESSAGE); return 0; } if (!strcmp(argument, "--help")) { usage_advanced(exeName); goto _cleanup; } - if (!strcmp(argument, "--keep")) { LZ4IO_setRemoveSrcFile(0); continue; } /* keep source file (default) */ - if (!strcmp(argument, "--rm")) { LZ4IO_setRemoveSrcFile(1); continue; } + if (!strcmp(argument, "--keep")) { LZ4IO_setRemoveSrcFile(prefs, 0); continue; } /* keep source file (default) */ + if (!strcmp(argument, "--rm")) { LZ4IO_setRemoveSrcFile(prefs, 1); continue; } if (longCommandWArg(&argument, "--fast")) { /* Parse optional acceleration factor */ if (*argument == '=') { @@ -406,7 +421,7 @@ int main(int argc, const char** argv) if (!strcmp(argument, "c1")) { cLevel=9; argument++; continue; } /* -c1 (high compression) */ if (!strcmp(argument, "c2")) { cLevel=12; argument++; continue; } /* -c2 (very high compression) */ if (!strcmp(argument, "hc")) { cLevel=12; argument++; continue; } /* -hc (very high compression) */ - if (!strcmp(argument, "y")) { LZ4IO_setOverwrite(1); continue; } /* -y (answer 'yes' to overwrite permission) */ + if (!strcmp(argument, "y")) { LZ4IO_setOverwrite(prefs, 1); continue; } /* -y (answer 'yes' to overwrite permission) */ } if ((*argument>='0') && (*argument<='9')) { @@ -455,13 +470,17 @@ int main(int argc, const char** argv) case 'd': mode = om_decompress; break; /* Force stdout, even if stdout==console */ - case 'c': forceStdout=1; output_filename=stdoutmark; break; + case 'c': + forceStdout=1; + output_filename=stdoutmark; + LZ4IO_setPassThrough(prefs, 1); + break; /* Test integrity */ case 't': mode = om_test; break; /* Overwrite */ - case 'f': LZ4IO_setOverwrite(1); break; + case 'f': LZ4IO_setOverwrite(prefs, 1); break; /* Verbose mode */ case 'v': displayLevel++; break; @@ -470,7 +489,7 @@ int main(int argc, const char** argv) case 'q': if (displayLevel) displayLevel--; break; /* keep source file (default anyway, so useless) (for xz/lzma compatibility) */ - case 'k': LZ4IO_setRemoveSrcFile(0); break; + case 'k': LZ4IO_setRemoveSrcFile(prefs, 0); break; /* Modify Block Properties */ case 'B': @@ -478,8 +497,8 @@ int main(int argc, const char** argv) int exitBlockProperties=0; switch(argument[1]) { - case 'D': LZ4IO_setBlockMode(LZ4IO_blockLinked); argument++; break; - case 'X': LZ4IO_setBlockChecksumMode(1); argument ++; break; /* disabled by default */ + case 'D': LZ4IO_setBlockMode(prefs, LZ4IO_blockLinked); argument++; break; + case 'X': LZ4IO_setBlockChecksumMode(prefs, 1); argument ++; break; /* disabled by default */ default : if (argument[1] < '0' || argument[1] > '9') { exitBlockProperties=1; @@ -491,12 +510,12 @@ int main(int argc, const char** argv) argument--; if (B < 4) badusage(exeName); if (B <= 7) { - blockSize = LZ4IO_setBlockSizeID(B); + blockSize = LZ4IO_setBlockSizeID(prefs, B); BMK_setBlockSize(blockSize); DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10)); } else { if (B < 32) badusage(exeName); - blockSize = LZ4IO_setBlockSize(B); + blockSize = LZ4IO_setBlockSize(prefs, B); BMK_setBlockSize(blockSize); if (blockSize >= 1024) { DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10)); @@ -605,7 +624,7 @@ int main(int argc, const char** argv) } if (mode == om_test) { - LZ4IO_setTestMode(1); + LZ4IO_setTestMode(prefs, 1); output_filename = nulmark; mode = om_decompress; /* defer to decompress */ } @@ -615,7 +634,7 @@ int main(int argc, const char** argv) DISPLAYLEVEL(1, "refusing to read from a console\n"); exit(1); } - LZ4IO_setDictionaryFilename(dictionary_filename); + LZ4IO_setDictionaryFilename(prefs, dictionary_filename); } /* compress or decompress */ @@ -633,11 +652,7 @@ int main(int argc, const char** argv) while ((!output_filename) && (multiple_inputs==0)) { if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; break; } /* Default to stdout whenever possible (i.e. not a console) */ if (mode == om_auto) { /* auto-determine compression or decompression, based on file extension */ - size_t const inSize = strlen(input_filename); - size_t const extSize = strlen(LZ4_EXTENSION); - size_t const extStart= (inSize > extSize) ? inSize-extSize : 0; - if (!strcmp(input_filename+extStart, LZ4_EXTENSION)) mode = om_decompress; - else mode = om_compress; + mode = determineOpMode(input_filename); } if (mode == om_compress) { /* compression to file */ size_t const l = strlen(input_filename); @@ -675,23 +690,28 @@ int main(int argc, const char** argv) if (!strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1; if ((multiple_inputs) && (displayLevel==2)) displayLevel=1; + /* Auto-determine compression or decompression, based on file extension */ + if (mode == om_auto) { + mode = determineOpMode(input_filename); + } + /* IO Stream/File */ LZ4IO_setNotificationLevel(displayLevel); if (ifnIdx == 0) multiple_inputs = 0; if (mode == om_decompress) { if (multiple_inputs) - operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, ifnIdx, !strcmp(output_filename,stdoutmark) ? stdoutmark : LZ4_EXTENSION); + operationResult = LZ4IO_decompressMultipleFilenames(prefs, inFileNames, ifnIdx, !strcmp(output_filename,stdoutmark) ? stdoutmark : LZ4_EXTENSION); else - operationResult = DEFAULT_DECOMPRESSOR(input_filename, output_filename); + operationResult = DEFAULT_DECOMPRESSOR(prefs, input_filename, output_filename); } else { /* compression is default action */ if (legacy_format) { DISPLAYLEVEL(3, "! Generating LZ4 Legacy format (deprecated) ! \n"); - LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel); + LZ4IO_compressFilename_Legacy(prefs, input_filename, output_filename, cLevel); } else { if (multiple_inputs) - operationResult = LZ4IO_compressMultipleFilenames(inFileNames, ifnIdx, LZ4_EXTENSION, cLevel); + operationResult = LZ4IO_compressMultipleFilenames(prefs, inFileNames, ifnIdx, LZ4_EXTENSION, cLevel); else - operationResult = DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel); + operationResult = DEFAULT_COMPRESSOR(prefs, input_filename, output_filename, cLevel); } } @@ -704,6 +724,7 @@ _cleanup: inFileNames = NULL; } #endif + LZ4IO_freePreferences(prefs); free((void*)inFileNames); return operationResult; } diff --git a/programs/lz4io.c b/programs/lz4io.c index a35928d..6bb6c48 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -107,19 +107,23 @@ static clock_t g_time = 0; /************************************** * Local Parameters **************************************/ -static int g_overwrite = 1; -static int g_testMode = 0; -static int g_blockSizeId = LZ4IO_BLOCKSIZEID_DEFAULT; -static size_t g_blockSize = 0; -static int g_blockChecksum = 0; -static int g_streamChecksum = 1; -static int g_blockIndependence = 1; -static int g_sparseFileSupport = 1; -static int g_contentSizeFlag = 0; -static int g_useDictionary = 0; -static unsigned g_favorDecSpeed = 0; -static const char* g_dictionaryFilename = NULL; +struct LZ4IO_prefs_s { + int passThrough; + int overwrite; + int testMode; + int blockSizeId; + size_t blockSize; + int blockChecksum; + int streamChecksum; + int blockIndependence; + int sparseFileSupport; + int contentSizeFlag; + int useDictionary; + unsigned favorDecSpeed; + char const* dictionaryFilename; + U32 removeSrcFile; +}; /************************************** * Exceptions @@ -151,73 +155,109 @@ static const char* g_dictionaryFilename = NULL; /* ****************** Parameters ******************** */ /* ************************************************** */ -int LZ4IO_setDictionaryFilename(const char* dictionaryFilename) { - g_dictionaryFilename = dictionaryFilename; - g_useDictionary = dictionaryFilename != NULL; - return g_useDictionary; +LZ4IO_prefs_t* LZ4IO_defaultPreferences(void) +{ + LZ4IO_prefs_t* const ret = (LZ4IO_prefs_t*)malloc(sizeof(LZ4IO_prefs_t)); + if (!ret) EXM_THROW(21, "Allocation error : not enough memory"); + ret->passThrough = 0; + ret->overwrite = 1; + ret->testMode = 0; + ret->blockSizeId = LZ4IO_BLOCKSIZEID_DEFAULT; + ret->blockSize = 0; + ret->blockChecksum = 0; + ret->streamChecksum = 1; + ret->blockIndependence = 1; + ret->sparseFileSupport = 1; + ret->contentSizeFlag = 0; + ret->useDictionary = 0; + ret->favorDecSpeed = 0; + ret->dictionaryFilename = NULL; + ret->removeSrcFile = 0; + return ret; +} + +void LZ4IO_freePreferences(LZ4IO_prefs_t* const prefs) +{ + free(prefs); +} + + +int LZ4IO_setDictionaryFilename(LZ4IO_prefs_t* const prefs, const char* dictionaryFilename) +{ + prefs->dictionaryFilename = dictionaryFilename; + prefs->useDictionary = dictionaryFilename != NULL; + return prefs->useDictionary; +} + +/* Default setting : passThrough = 0; return : passThrough mode (0/1) */ +int LZ4IO_setPassThrough(LZ4IO_prefs_t* const prefs, int yes) +{ + prefs->passThrough = (yes!=0); + return prefs->passThrough; } + /* Default setting : overwrite = 1; return : overwrite mode (0/1) */ -int LZ4IO_setOverwrite(int yes) +int LZ4IO_setOverwrite(LZ4IO_prefs_t* const prefs, int yes) { - g_overwrite = (yes!=0); - return g_overwrite; + prefs->overwrite = (yes!=0); + return prefs->overwrite; } /* Default setting : testMode = 0; return : testMode (0/1) */ -int LZ4IO_setTestMode(int yes) +int LZ4IO_setTestMode(LZ4IO_prefs_t* const prefs, int yes) { - g_testMode = (yes!=0); - return g_testMode; + prefs->testMode = (yes!=0); + return prefs->testMode; } /* blockSizeID : valid values : 4-5-6-7 */ -size_t LZ4IO_setBlockSizeID(unsigned bsid) +size_t LZ4IO_setBlockSizeID(LZ4IO_prefs_t* const prefs, unsigned bsid) { static const size_t blockSizeTable[] = { 64 KB, 256 KB, 1 MB, 4 MB }; static const unsigned minBlockSizeID = 4; static const unsigned maxBlockSizeID = 7; if ((bsid < minBlockSizeID) || (bsid > maxBlockSizeID)) return 0; - g_blockSizeId = bsid; - g_blockSize = blockSizeTable[g_blockSizeId-minBlockSizeID]; - return g_blockSize; + prefs->blockSizeId = bsid; + prefs->blockSize = blockSizeTable[prefs->blockSizeId-minBlockSizeID]; + return prefs->blockSize; } -size_t LZ4IO_setBlockSize(size_t blockSize) +size_t LZ4IO_setBlockSize(LZ4IO_prefs_t* const prefs, size_t blockSize) { static const size_t minBlockSize = 32; static const size_t maxBlockSize = 4 MB; unsigned bsid = 0; if (blockSize < minBlockSize) blockSize = minBlockSize; if (blockSize > maxBlockSize) blockSize = maxBlockSize; - g_blockSize = blockSize; + prefs->blockSize = blockSize; blockSize--; /* find which of { 64k, 256k, 1MB, 4MB } is closest to blockSize */ while (blockSize >>= 2) bsid++; if (bsid < 7) bsid = 7; - g_blockSizeId = bsid-3; - return g_blockSize; + prefs->blockSizeId = bsid-3; + return prefs->blockSize; } -int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode) +int LZ4IO_setBlockMode(LZ4IO_prefs_t* const prefs, LZ4IO_blockMode_t blockMode) { - g_blockIndependence = (blockMode == LZ4IO_blockIndependent); - return g_blockIndependence; + prefs->blockIndependence = (blockMode == LZ4IO_blockIndependent); + return prefs->blockIndependence; } /* Default setting : no block checksum */ -int LZ4IO_setBlockChecksumMode(int enable) +int LZ4IO_setBlockChecksumMode(LZ4IO_prefs_t* const prefs, int enable) { - g_blockChecksum = (enable != 0); - return g_blockChecksum; + prefs->blockChecksum = (enable != 0); + return prefs->blockChecksum; } /* Default setting : checksum enabled */ -int LZ4IO_setStreamChecksumMode(int enable) +int LZ4IO_setStreamChecksumMode(LZ4IO_prefs_t* const prefs, int enable) { - g_streamChecksum = (enable != 0); - return g_streamChecksum; + prefs->streamChecksum = (enable != 0); + return prefs->streamChecksum; } /* Default setting : 0 (no notification) */ @@ -228,27 +268,29 @@ int LZ4IO_setNotificationLevel(int level) } /* Default setting : 0 (disabled) */ -int LZ4IO_setSparseFile(int enable) +int LZ4IO_setSparseFile(LZ4IO_prefs_t* const prefs, int enable) { - g_sparseFileSupport = (enable!=0); - return g_sparseFileSupport; + prefs->sparseFileSupport = (enable!=0); + return prefs->sparseFileSupport; } /* Default setting : 0 (disabled) */ -int LZ4IO_setContentSize(int enable) +int LZ4IO_setContentSize(LZ4IO_prefs_t* const prefs, int enable) { - g_contentSizeFlag = (enable!=0); - return g_contentSizeFlag; + prefs->contentSizeFlag = (enable!=0); + return prefs->contentSizeFlag; } /* Default setting : 0 (disabled) */ -void LZ4IO_favorDecSpeed(int favor) +void LZ4IO_favorDecSpeed(LZ4IO_prefs_t* const prefs, int favor) { - g_favorDecSpeed = (favor!=0); + prefs->favorDecSpeed = (favor!=0); } -static U32 g_removeSrcFile = 0; -void LZ4IO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); } +void LZ4IO_setRemoveSrcFile(LZ4IO_prefs_t* const prefs, unsigned flag) +{ + prefs->removeSrcFile = (flag>0); +} @@ -283,7 +325,7 @@ static FILE* LZ4IO_openSrcFile(const char* srcFileName) /** FIO_openDstFile() : * condition : `dstFileName` must be non-NULL. * @result : FILE* to `dstFileName`, or NULL if it fails */ -static FILE* LZ4IO_openDstFile(const char* dstFileName) +static FILE* LZ4IO_openDstFile(LZ4IO_prefs_t* const prefs, const char* dstFileName) { FILE* f; @@ -291,12 +333,12 @@ static FILE* LZ4IO_openDstFile(const char* dstFileName) DISPLAYLEVEL(4,"Using stdout for output\n"); f = stdout; SET_BINARY_MODE(stdout); - if (g_sparseFileSupport==1) { - g_sparseFileSupport = 0; + if (prefs->sparseFileSupport==1) { + prefs->sparseFileSupport = 0; DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); } } else { - if (!g_overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */ + if (!prefs->overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */ f = fopen( dstFileName, "rb" ); if (f != NULL) { /* dest exists, prompt for overwrite authorization */ fclose(f); @@ -317,7 +359,7 @@ static FILE* LZ4IO_openDstFile(const char* dstFileName) } /* sparse file */ - if (f && g_sparseFileSupport) { SET_SPARSE_FILE_MODE(f); } + if (f && prefs->sparseFileSupport) { SET_SPARSE_FILE_MODE(f); } return f; } @@ -347,7 +389,7 @@ static int LZ4IO_LZ4_compress(const char* src, char* dst, int srcSize, int dstSi /* LZ4IO_compressFilename_Legacy : * This function is intentionally "hidden" (not published in .h) * It generates compressed streams using the old 'legacy' format */ -int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel) +int LZ4IO_compressFilename_Legacy(LZ4IO_prefs_t* const prefs, const char* input_filename, const char* output_filename, int compressionlevel) { typedef int (*compress_f)(const char* src, char* dst, int srcSize, int dstSize, int cLevel); compress_f const compressionFunction = (compressionlevel < 3) ? LZ4IO_LZ4_compress : LZ4_compress_HC; @@ -365,7 +407,7 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output if (finput == NULL) EXM_THROW(20, "%s : open file error ", input_filename); - foutput = LZ4IO_openDstFile(output_filename); + foutput = LZ4IO_openDstFile(prefs, output_filename); if (foutput == NULL) { fclose(finput); EXM_THROW(20, "%s : open file error ", input_filename); @@ -444,7 +486,7 @@ typedef struct { LZ4F_CDict* cdict; } cRess_t; -static void* LZ4IO_createDict(const char* dictFilename, size_t *dictSize) { +static void* LZ4IO_createDict(LZ4IO_prefs_t* const prefs, size_t *dictSize) { size_t readSize; size_t dictEnd = 0; size_t dictLen = 0; @@ -452,6 +494,7 @@ static void* LZ4IO_createDict(const char* dictFilename, size_t *dictSize) { size_t circularBufSize = LZ4_MAX_DICT_SIZE; char* circularBuf; char* dictBuf; + const char* dictFilename = prefs->dictionaryFilename; FILE* dictFile; if (!dictFilename) EXM_THROW(25, "Dictionary error : no filename provided"); @@ -501,23 +544,23 @@ static void* LZ4IO_createDict(const char* dictFilename, size_t *dictSize) { return dictBuf; } -static LZ4F_CDict* LZ4IO_createCDict(void) { +static LZ4F_CDict* LZ4IO_createCDict(LZ4IO_prefs_t* const prefs) { size_t dictionarySize; void* dictionaryBuffer; LZ4F_CDict* cdict; - if (!g_useDictionary) { + if (!prefs->useDictionary) { return NULL; } - dictionaryBuffer = LZ4IO_createDict(g_dictionaryFilename, &dictionarySize); + dictionaryBuffer = LZ4IO_createDict(prefs, &dictionarySize); if (!dictionaryBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary"); cdict = LZ4F_createCDict(dictionaryBuffer, dictionarySize); free(dictionaryBuffer); return cdict; } -static cRess_t LZ4IO_createCResources(void) +static cRess_t LZ4IO_createCResources(LZ4IO_prefs_t* const prefs) { - const size_t blockSize = g_blockSize; + const size_t blockSize = prefs->blockSize; cRess_t ress; LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&(ress.ctx), LZ4F_VERSION); @@ -530,7 +573,7 @@ static cRess_t LZ4IO_createCResources(void) ress.dstBuffer = malloc(ress.dstBufferSize); if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory"); - ress.cdict = LZ4IO_createCDict(); + ress.cdict = LZ4IO_createCDict(prefs); return ress; } @@ -552,7 +595,7 @@ static void LZ4IO_freeCResources(cRess_t ress) * result : 0 : compression completed correctly * 1 : missing or pb opening srcFileName */ -static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, const char* dstFileName, int compressionLevel) +static int LZ4IO_compressFilename_extRess(LZ4IO_prefs_t* const io_prefs, cRess_t ress, const char* srcFileName, const char* dstFileName, int compressionLevel) { unsigned long long filesize = 0; unsigned long long compressedfilesize = 0; @@ -561,7 +604,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, void* const srcBuffer = ress.srcBuffer; void* const dstBuffer = ress.dstBuffer; const size_t dstBufferSize = ress.dstBufferSize; - const size_t blockSize = g_blockSize; + const size_t blockSize = io_prefs->blockSize; size_t readSize; LZ4F_compressionContext_t ctx = ress.ctx; /* just a pointer */ LZ4F_preferences_t prefs; @@ -569,7 +612,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, /* Init */ srcFile = LZ4IO_openSrcFile(srcFileName); if (srcFile == NULL) return 1; - dstFile = LZ4IO_openDstFile(dstFileName); + dstFile = LZ4IO_openDstFile(io_prefs, dstFileName); if (dstFile == NULL) { fclose(srcFile); return 1; } memset(&prefs, 0, sizeof(prefs)); @@ -577,12 +620,12 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, /* Set compression parameters */ prefs.autoFlush = 1; prefs.compressionLevel = compressionLevel; - prefs.frameInfo.blockMode = (LZ4F_blockMode_t)g_blockIndependence; - prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)g_blockSizeId; - prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)g_blockChecksum; - prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)g_streamChecksum; - prefs.favorDecSpeed = g_favorDecSpeed; - if (g_contentSizeFlag) { + prefs.frameInfo.blockMode = (LZ4F_blockMode_t)io_prefs->blockIndependence; + prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)io_prefs->blockSizeId; + prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)io_prefs->blockChecksum; + prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)io_prefs->streamChecksum; + prefs.favorDecSpeed = io_prefs->favorDecSpeed; + if (io_prefs->contentSizeFlag) { U64 const fileSize = UTIL_getFileSize(srcFileName); prefs.frameInfo.contentSize = fileSize; /* == 0 if input == stdin */ if (fileSize==0) @@ -661,7 +704,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, UTIL_setFileStat(dstFileName, &statbuf); } } - if (g_removeSrcFile) { /* remove source file : --rm */ + if (io_prefs->removeSrcFile) { /* remove source file : --rm */ if (remove(srcFileName)) EXM_THROW(40, "Remove error : %s: %s", srcFileName, strerror(errno)); } @@ -676,13 +719,13 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, } -int LZ4IO_compressFilename(const char* srcFileName, const char* dstFileName, int compressionLevel) +int LZ4IO_compressFilename(LZ4IO_prefs_t* const prefs, const char* srcFileName, const char* dstFileName, int compressionLevel) { UTIL_time_t const timeStart = UTIL_getTime(); clock_t const cpuStart = clock(); - cRess_t const ress = LZ4IO_createCResources(); + cRess_t const ress = LZ4IO_createCResources(prefs); - int const result = LZ4IO_compressFilename_extRess(ress, srcFileName, dstFileName, compressionLevel); + int const result = LZ4IO_compressFilename_extRess(prefs, ress, srcFileName, dstFileName, compressionLevel); /* Free resources */ LZ4IO_freeCResources(ress); @@ -701,7 +744,7 @@ int LZ4IO_compressFilename(const char* srcFileName, const char* dstFileName, int #define FNSPACE 30 -int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionLevel) +int LZ4IO_compressMultipleFilenames(LZ4IO_prefs_t* const prefs, const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionLevel) { int i; int missed_files = 0; @@ -711,7 +754,7 @@ int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, cRess_t ress; if (dstFileName == NULL) return ifntSize; /* not enough memory */ - ress = LZ4IO_createCResources(); + ress = LZ4IO_createCResources(prefs); /* loop on each file */ for (i=0; i<ifntSize; i++) { @@ -720,7 +763,7 @@ int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, strcpy(dstFileName, inFileNamesTable[i]); strcat(dstFileName, suffix); - missed_files += LZ4IO_compressFilename_extRess(ress, inFileNamesTable[i], dstFileName, compressionLevel); + missed_files += LZ4IO_compressFilename_extRess(prefs, ress, inFileNamesTable[i], dstFileName, compressionLevel); } /* Close & Free */ @@ -746,7 +789,7 @@ static unsigned LZ4IO_readLE32 (const void* s) } -static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips) +static unsigned LZ4IO_fwriteSparse(LZ4IO_prefs_t* const prefs, FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips) { const size_t sizeT = sizeof(size_t); const size_t maskT = sizeT -1 ; @@ -756,7 +799,7 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer const size_t* const bufferTEnd = bufferT + bufferSizeT; const size_t segmentSizeT = (32 KB) / sizeT; - if (!g_sparseFileSupport) { /* normal write */ + if (!prefs->sparseFileSupport) { /* normal write */ size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : cannot write decoded block"); return 0; @@ -825,7 +868,7 @@ static void LZ4IO_fwriteSparseEnd(FILE* file, unsigned storedSkips) static unsigned g_magicRead = 0; /* out-parameter of LZ4IO_decodeLegacyStream() */ -static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput) +static unsigned long long LZ4IO_decodeLegacyStream(LZ4IO_prefs_t* const prefs, FILE* finput, FILE* foutput) { unsigned long long streamSize = 0; unsigned storedSkips = 0; @@ -859,7 +902,7 @@ static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput) if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !"); streamSize += decodeSize; /* Write Block */ - storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, decodeSize, storedSkips); /* success or die */ + storedSkips = LZ4IO_fwriteSparse(prefs, foutput, out_buff, decodeSize, storedSkips); /* success or die */ } } if (ferror(finput)) EXM_THROW(54, "Read error : ferror"); @@ -885,19 +928,19 @@ typedef struct { size_t dictBufferSize; } dRess_t; -static void LZ4IO_loadDDict(dRess_t* ress) { - if (!g_useDictionary) { +static void LZ4IO_loadDDict(LZ4IO_prefs_t* const prefs, dRess_t* ress) { + if (!prefs->useDictionary) { ress->dictBuffer = NULL; ress->dictBufferSize = 0; return; } - ress->dictBuffer = LZ4IO_createDict(g_dictionaryFilename, &ress->dictBufferSize); + ress->dictBuffer = LZ4IO_createDict(prefs, &ress->dictBufferSize); if (!ress->dictBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary"); } static const size_t LZ4IO_dBufferSize = 64 KB; -static dRess_t LZ4IO_createDResources(void) +static dRess_t LZ4IO_createDResources(LZ4IO_prefs_t* const prefs) { dRess_t ress; @@ -912,7 +955,7 @@ static dRess_t LZ4IO_createDResources(void) ress.dstBuffer = malloc(ress.dstBufferSize); if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory"); - LZ4IO_loadDDict(&ress); + LZ4IO_loadDDict(prefs, &ress); ress.dstFile = NULL; return ress; @@ -928,7 +971,7 @@ static void LZ4IO_freeDResources(dRess_t ress) } -static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE* dstFile) +static unsigned long long LZ4IO_decompressLZ4F(LZ4IO_prefs_t* const prefs, dRess_t ress, FILE* srcFile, FILE* dstFile) { unsigned long long filesize = 0; LZ4F_errorCode_t nextToLoad; @@ -963,8 +1006,8 @@ static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE /* Write Block */ if (decodedBytes) { - if (!g_testMode) - storedSkips = LZ4IO_fwriteSparse(dstFile, ress.dstBuffer, decodedBytes, storedSkips); + if (!prefs->testMode) + storedSkips = LZ4IO_fwriteSparse(prefs, dstFile, ress.dstBuffer, decodedBytes, storedSkips); filesize += decodedBytes; DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20)); } @@ -975,7 +1018,7 @@ static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE /* can be out because readSize == 0, which could be an fread() error */ if (ferror(srcFile)) EXM_THROW(67, "Read error"); - if (!g_testMode) LZ4IO_fwriteSparseEnd(dstFile, storedSkips); + if (!prefs->testMode) LZ4IO_fwriteSparseEnd(dstFile, storedSkips); if (nextToLoad!=0) EXM_THROW(68, "Unfinished stream"); return filesize; @@ -984,7 +1027,7 @@ static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE #define PTSIZE (64 KB) #define PTSIZET (PTSIZE / sizeof(size_t)) -static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigned char MNstore[MAGICNUMBER_SIZE]) +static unsigned long long LZ4IO_passThrough(LZ4IO_prefs_t* const prefs, FILE* finput, FILE* foutput, unsigned char MNstore[MAGICNUMBER_SIZE]) { size_t buffer[PTSIZET]; size_t readBytes = 1; @@ -997,7 +1040,7 @@ static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigne while (readBytes) { readBytes = fread(buffer, 1, PTSIZE, finput); total += readBytes; - storedSkips = LZ4IO_fwriteSparse(foutput, buffer, readBytes, storedSkips); + storedSkips = LZ4IO_fwriteSparse(prefs, foutput, buffer, readBytes, storedSkips); } if (ferror(finput)) EXM_THROW(51, "Read Error"); @@ -1024,7 +1067,7 @@ static int fseek_u32(FILE *fp, unsigned offset, int where) } #define ENDOFSTREAM ((unsigned long long)-1) -static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutput) +static unsigned long long selectDecoder(LZ4IO_prefs_t* const prefs, dRess_t ress, FILE* finput, FILE* foutput) { unsigned char MNstore[MAGICNUMBER_SIZE]; unsigned magicNumber; @@ -1050,10 +1093,10 @@ static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutpu switch(magicNumber) { case LZ4IO_MAGICNUMBER: - return LZ4IO_decompressLZ4F(ress, finput, foutput); + return LZ4IO_decompressLZ4F(prefs, ress, finput, foutput); case LEGACY_MAGICNUMBER: DISPLAYLEVEL(4, "Detected : Legacy format \n"); - return LZ4IO_decodeLegacyStream(finput, foutput); + return LZ4IO_decodeLegacyStream(prefs, finput, foutput); case LZ4IO_SKIPPABLE0: DISPLAYLEVEL(4, "Skipping detected skippable area \n"); { size_t const nbReadBytes = fread(MNstore, 1, 4, finput); @@ -1070,9 +1113,9 @@ static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutpu default: if (nbFrames == 1) { /* just started */ /* Wrong magic number at the beginning of 1st stream */ - if (!g_testMode && g_overwrite) { + if (!prefs->testMode && prefs->overwrite && prefs->passThrough) { nbFrames = 0; - return LZ4IO_passThrough(finput, foutput, MNstore); + return LZ4IO_passThrough(prefs, finput, foutput, MNstore); } EXM_THROW(44,"Unrecognized header : file cannot be decoded"); } @@ -1087,7 +1130,7 @@ static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutpu } -static int LZ4IO_decompressSrcFile(dRess_t ress, const char* input_filename, const char* output_filename) +static int LZ4IO_decompressSrcFile(LZ4IO_prefs_t* const prefs, dRess_t ress, const char* input_filename, const char* output_filename) { FILE* const foutput = ress.dstFile; unsigned long long filesize = 0; @@ -1099,14 +1142,14 @@ static int LZ4IO_decompressSrcFile(dRess_t ress, const char* input_filename, con /* Loop over multiple streams */ for ( ; ; ) { /* endless loop, see break condition */ unsigned long long const decodedSize = - selectDecoder(ress, finput, foutput); + selectDecoder(prefs, ress, finput, foutput); if (decodedSize == ENDOFSTREAM) break; filesize += decodedSize; } /* Close input */ fclose(finput); - if (g_removeSrcFile) { /* --rm */ + if (prefs->removeSrcFile) { /* --rm */ if (remove(input_filename)) EXM_THROW(45, "Remove error : %s: %s", input_filename, strerror(errno)); } @@ -1120,11 +1163,11 @@ static int LZ4IO_decompressSrcFile(dRess_t ress, const char* input_filename, con } -static int LZ4IO_decompressDstFile(dRess_t ress, const char* input_filename, const char* output_filename) +static int LZ4IO_decompressDstFile(LZ4IO_prefs_t* const prefs, dRess_t ress, const char* input_filename, const char* output_filename) { stat_t statbuf; int stat_result = 0; - FILE* const foutput = LZ4IO_openDstFile(output_filename); + FILE* const foutput = LZ4IO_openDstFile(prefs, output_filename); if (foutput==NULL) return 1; /* failure */ if ( strcmp(input_filename, stdinmark) @@ -1132,7 +1175,7 @@ static int LZ4IO_decompressDstFile(dRess_t ress, const char* input_filename, con stat_result = 1; ress.dstFile = foutput; - LZ4IO_decompressSrcFile(ress, input_filename, output_filename); + LZ4IO_decompressSrcFile(prefs, ress, input_filename, output_filename); fclose(foutput); @@ -1148,12 +1191,12 @@ static int LZ4IO_decompressDstFile(dRess_t ress, const char* input_filename, con } -int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename) +int LZ4IO_decompressFilename(LZ4IO_prefs_t* const prefs, const char* input_filename, const char* output_filename) { - dRess_t const ress = LZ4IO_createDResources(); + dRess_t const ress = LZ4IO_createDResources(prefs); clock_t const start = clock(); - int const missingFiles = LZ4IO_decompressDstFile(ress, input_filename, output_filename); + int const missingFiles = LZ4IO_decompressDstFile(prefs, ress, input_filename, output_filename); clock_t const end = clock(); double const seconds = (double)(end - start) / CLOCKS_PER_SEC; @@ -1164,7 +1207,7 @@ int LZ4IO_decompressFilename(const char* input_filename, const char* output_file } -int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix) +int LZ4IO_decompressMultipleFilenames(LZ4IO_prefs_t* const prefs, const char** inFileNamesTable, int ifntSize, const char* suffix) { int i; int skippedFiles = 0; @@ -1172,16 +1215,16 @@ int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSiz char* outFileName = (char*)malloc(FNSPACE); size_t ofnSize = FNSPACE; size_t const suffixSize = strlen(suffix); - dRess_t ress = LZ4IO_createDResources(); + dRess_t ress = LZ4IO_createDResources(prefs); if (outFileName==NULL) return ifntSize; /* not enough memory */ - ress.dstFile = LZ4IO_openDstFile(stdoutmark); + ress.dstFile = LZ4IO_openDstFile(prefs, stdoutmark); for (i=0; i<ifntSize; i++) { size_t const ifnSize = strlen(inFileNamesTable[i]); const char* const suffixPtr = inFileNamesTable[i] + ifnSize - suffixSize; if (!strcmp(suffix, stdoutmark)) { - missingFiles += LZ4IO_decompressSrcFile(ress, inFileNamesTable[i], stdoutmark); + missingFiles += LZ4IO_decompressSrcFile(prefs, ress, inFileNamesTable[i], stdoutmark); continue; } if (ofnSize <= ifnSize-suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); if (outFileName==NULL) return ifntSize; } @@ -1192,7 +1235,7 @@ int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSiz } memcpy(outFileName, inFileNamesTable[i], ifnSize - suffixSize); outFileName[ifnSize-suffixSize] = '\0'; - missingFiles += LZ4IO_decompressDstFile(ress, inFileNamesTable[i], outFileName); + missingFiles += LZ4IO_decompressDstFile(prefs, ress, inFileNamesTable[i], outFileName); } LZ4IO_freeDResources(ress); diff --git a/programs/lz4io.h b/programs/lz4io.h index 33de41f..0c28784 100644 --- a/programs/lz4io.h +++ b/programs/lz4io.h @@ -48,65 +48,77 @@ static const char nulmark[] = "nul"; static const char nulmark[] = "/dev/null"; #endif +/* ************************************************** */ +/* ****************** Type Definitions ************** */ +/* ************************************************** */ + +typedef struct LZ4IO_prefs_s LZ4IO_prefs_t; + +LZ4IO_prefs_t* LZ4IO_defaultPreferences(void); +void LZ4IO_freePreferences(LZ4IO_prefs_t* const prefs); /* ************************************************** */ /* ****************** Functions ********************* */ /* ************************************************** */ -int LZ4IO_compressFilename (const char* input_filename, const char* output_filename, int compressionlevel); -int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename); +int LZ4IO_compressFilename(LZ4IO_prefs_t* const prefs, const char* input_filename, const char* output_filename, int compressionlevel); +int LZ4IO_decompressFilename(LZ4IO_prefs_t* const prefs, const char* input_filename, const char* output_filename); -int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel); -int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix); +int LZ4IO_compressMultipleFilenames(LZ4IO_prefs_t* const prefs, const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel); +int LZ4IO_decompressMultipleFilenames(LZ4IO_prefs_t* const prefs, const char** inFileNamesTable, int ifntSize, const char* suffix); /* ************************************************** */ /* ****************** Parameters ******************** */ /* ************************************************** */ -int LZ4IO_setDictionaryFilename(const char* dictionaryFilename); +int LZ4IO_setDictionaryFilename(LZ4IO_prefs_t* const prefs, const char* dictionaryFilename); + +/* Default setting : passThrough = 0; + return : passThrough mode (0/1) */ +int LZ4IO_setPassThrough(LZ4IO_prefs_t* const prefs, int yes); /* Default setting : overwrite = 1; return : overwrite mode (0/1) */ -int LZ4IO_setOverwrite(int yes); +int LZ4IO_setOverwrite(LZ4IO_prefs_t* const prefs, int yes); /* Default setting : testMode = 0; return : testMode (0/1) */ -int LZ4IO_setTestMode(int yes); +int LZ4IO_setTestMode(LZ4IO_prefs_t* const prefs, int yes); /* blockSizeID : valid values : 4-5-6-7 return : 0 if error, blockSize if OK */ -size_t LZ4IO_setBlockSizeID(unsigned blockSizeID); +size_t LZ4IO_setBlockSizeID(LZ4IO_prefs_t* const prefs, unsigned blockSizeID); /* blockSize : valid values : 32 -> 4MB return : 0 if error, actual blocksize if OK */ -size_t LZ4IO_setBlockSize(size_t blockSize); +size_t LZ4IO_setBlockSize(LZ4IO_prefs_t* const prefs, size_t blockSize); /* Default setting : independent blocks */ typedef enum { LZ4IO_blockLinked=0, LZ4IO_blockIndependent} LZ4IO_blockMode_t; -int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode); +int LZ4IO_setBlockMode(LZ4IO_prefs_t* const prefs, LZ4IO_blockMode_t blockMode); /* Default setting : no block checksum */ -int LZ4IO_setBlockChecksumMode(int xxhash); +int LZ4IO_setBlockChecksumMode(LZ4IO_prefs_t* const prefs, int xxhash); /* Default setting : stream checksum enabled */ -int LZ4IO_setStreamChecksumMode(int xxhash); +int LZ4IO_setStreamChecksumMode(LZ4IO_prefs_t* const prefs, int xxhash); /* Default setting : 0 (no notification) */ int LZ4IO_setNotificationLevel(int level); /* Default setting : 0 (disabled) */ -int LZ4IO_setSparseFile(int enable); +int LZ4IO_setSparseFile(LZ4IO_prefs_t* const prefs, int enable); /* Default setting : 0 == no content size present in frame header */ -int LZ4IO_setContentSize(int enable); +int LZ4IO_setContentSize(LZ4IO_prefs_t* const prefs, int enable); /* Default setting : 0 == src file preserved */ -void LZ4IO_setRemoveSrcFile(unsigned flag); +void LZ4IO_setRemoveSrcFile(LZ4IO_prefs_t* const prefs, unsigned flag); /* Default setting : 0 == favor compression ratio * Note : 1 only works for high compression levels (10+) */ -void LZ4IO_favorDecSpeed(int favor); +void LZ4IO_favorDecSpeed(LZ4IO_prefs_t* const prefs, int favor); #endif /* LZ4IO_H_237902873 */ diff --git a/tests/.gitignore b/tests/.gitignore index 9aa42a0..c4f9092 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -10,6 +10,7 @@ fuzzer32 fasttest roundTripTest checkTag +checkFrame # test artefacts tmp* diff --git a/tests/Makefile b/tests/Makefile index 8dcef6d..7d49b31 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -284,6 +284,11 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat test "$(shell ./datagen -g20KB | $(LZ4) -c --fast=1 | wc -c)" -eq "$(shell ./datagen -g20KB| $(LZ4) -c --fast| wc -c)" # checks default fast compression is -1 ! $(LZ4) -c --fast=0 tmp-tlb-dg20K # lz4 should fail when fast=0 ! $(LZ4) -c --fast=-1 tmp-tlb-dg20K # lz4 should fail when fast=-1 + # Test for #596 + @echo "TEST" > tmp-tlb-test + $(LZ4) tmp-tlb-test + $(LZ4) tmp-tlb-test.lz4 tmp-tlb-test2 + $(DIFF) -q tmp-tlb-test tmp-tlb-test2 @$(RM) tmp-tlb* @@ -332,8 +337,11 @@ test-lz4-testmode: lz4 datagen ! ./datagen | $(LZ4) -t ! ./datagen | $(LZ4) -tf @echo "\n ---- pass-through mode ----" - ! ./datagen | $(LZ4) -d > $(VOID) - ./datagen | $(LZ4) -df > $(VOID) + @echo "Why hello there " > tmp-tlt2.lz4 + ! $(LZ4) -f tmp-tlt2.lz4 > $(VOID) + ! ./datagen | $(LZ4) -dc > $(VOID) + ! ./datagen | $(LZ4) -df > $(VOID) + ./datagen | $(LZ4) -dcf > $(VOID) @echo "Hello World !" > tmp-tlt1 $(LZ4) -dcf tmp-tlt1 @echo "from underground..." > tmp-tlt2 @@ -342,7 +350,7 @@ test-lz4-testmode: lz4 datagen ! $(LZ4) file-does-not-exist ! $(LZ4) -f file-does-not-exist ! $(LZ4) -fm file1-dne file2-dne - @$(RM) tmp-tlt + @$(RM) tmp-tlt tmp-tlt1 tmp-tlt2 tmp-tlt2.lz4 test-lz4-opt-parser: lz4 datagen @echo "\n ---- test opt-parser ----" diff --git a/tests/frametest.c b/tests/frametest.c index b93f4fe..14906a0 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -658,6 +658,29 @@ int basicTests(U32 seed, double compressibility) CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL; } + DISPLAYLEVEL(3, "getBlockSize test: \n"); + { size_t result; + unsigned blockSizeID; + for (blockSizeID = 4; blockSizeID < 8; ++blockSizeID) { + result = LZ4F_getBlockSize(blockSizeID); + CHECK(result); + DISPLAYLEVEL(3, "Returned block size of %zu bytes for blockID %u \n", + result, blockSizeID); + } + + /* Test an invalid input that's too large */ + result = LZ4F_getBlockSize(8); + if(!LZ4F_isError(result) || + LZ4F_getErrorCode(result) != LZ4F_ERROR_maxBlockSize_invalid) + goto _output_error; + + /* Test an invalid input that's too small */ + result = LZ4F_getBlockSize(3); + if(!LZ4F_isError(result) || + LZ4F_getErrorCode(result) != LZ4F_ERROR_maxBlockSize_invalid) + goto _output_error; + } + DISPLAYLEVEL(3, "Skippable frame test : \n"); { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; diff --git a/tests/fuzzer.c b/tests/fuzzer.c index fab1fce..893ffc1 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -48,6 +48,7 @@ #include <string.h> /* strcmp */ #include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */ #include <assert.h> +#include <limits.h> /* INT_MAX */ #if defined(__unix__) && defined(_AIX) # include <sys/mman.h> /* mmap */ #endif @@ -1064,45 +1065,81 @@ static void FUZ_unitTests(int compressionLevel) } /* LZ4 HC streaming tests */ - { LZ4_streamHC_t* sp; - LZ4_streamHC_t sHC; + { LZ4_streamHC_t sHC; /* statically allocated */ U64 crcOrig; int result; /* Allocation test */ - sp = LZ4_createStreamHC(); - FUZ_CHECKTEST(sp==NULL, "LZ4_createStreamHC() allocation failed"); - LZ4_freeStreamHC(sp); + DISPLAYLEVEL(3, " Basic HC allocation : "); + { LZ4_streamHC_t* const sp = LZ4_createStreamHC(); + FUZ_CHECKTEST(sp==NULL, "LZ4_createStreamHC() allocation failed"); + LZ4_freeStreamHC(sp); + } + DISPLAYLEVEL(3, " OK \n"); /* simple HC compression test */ - crcOrig = XXH64(testInput, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, compressionLevel); - result = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); - FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); + DISPLAYLEVEL(3, " Simple HC round-trip : "); + { U64 const crc64 = XXH64(testInput, testCompressedSize, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); + result = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1); + FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); - result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); - { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); } + result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize); + FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); + { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() decompression corruption"); + } } + DISPLAYLEVEL(3, " OK \n"); + + /* long sequence test */ + DISPLAYLEVEL(3, " Long sequence HC test : "); + { size_t const blockSize = 1 MB; + size_t const targetSize = 4116; /* size carefully selected to trigger an overflow */ + void* const block = malloc(blockSize); + void* const dstBlock = malloc(targetSize+1); + BYTE const sentinel = 101; + int srcSize; + + assert(block != NULL); assert(dstBlock != NULL); + memset(block, 0, blockSize); + ((char*)dstBlock)[targetSize] = sentinel; + + LZ4_resetStreamHC(&sHC, 3); + assert(blockSize < INT_MAX); + srcSize = (int)blockSize; + assert(targetSize < INT_MAX); + result = LZ4_compress_HC_destSize(&sHC, (const char*)block, (char*)dstBlock, &srcSize, (int)targetSize, 3); + DISPLAYLEVEL(4, "cSize=%i; readSize=%i; ", result, srcSize); + FUZ_CHECKTEST(result!=4116, "LZ4_compress_HC_destSize() : compression must fill dstBuffer completely, but no more !"); + FUZ_CHECKTEST(((char*)dstBlock)[targetSize] != sentinel, "LZ4_compress_HC_destSize()") + + LZ4_resetStreamHC(&sHC, 3); /* make sure the context is clean after the test */ + free(block); + free(dstBlock); + } + DISPLAYLEVEL(3, " OK \n"); /* simple dictionary HC compression test */ - crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ4_resetStreamHC_fast(&sHC, compressionLevel); - LZ4_loadDictHC(&sHC, testInput, 64 KB); - result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); - FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); - - result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 64 KB); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed"); - { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() simple dictionary decompression test : corruption"); } + DISPLAYLEVEL(3, " HC dictionary compression test : "); + { U64 const crc64 = XXH64(testInput + 64 KB, testCompressedSize, 0); + LZ4_resetStreamHC_fast(&sHC, compressionLevel); + LZ4_loadDictHC(&sHC, testInput, 64 KB); + result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); + FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); + + result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 64 KB); + FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed"); + { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() simple dictionary decompression test : corruption"); + } } + DISPLAYLEVEL(3, " OK \n"); /* multiple HC compression test with dictionary */ { int result1, result2; int segSize = testCompressedSize / 2; - crcOrig = XXH64(testInput + segSize, testCompressedSize, 0); + U64 const crc64 = XXH64(testInput + segSize, testCompressedSize, 0); LZ4_resetStreamHC_fast(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, segSize); result1 = LZ4_compress_HC_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1); @@ -1116,22 +1153,23 @@ static void FUZ_unitTests(int compressionLevel) FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 1 failed"); result = LZ4_decompress_safe_usingDict(testCompressed+result1, testVerify+segSize, result2, segSize, testInput, 2*segSize); FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 2 failed"); - { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); } - } + { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); + } } /* remote dictionary HC compression test */ - crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ4_resetStreamHC_fast(&sHC, compressionLevel); - LZ4_loadDictHC(&sHC, testInput, 32 KB); - result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result); - FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); - - result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 32 KB); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe_usingDict() decompression failed following remote dictionary HC compression test"); - { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() decompression corruption"); } + { U64 const crc64 = XXH64(testInput + 64 KB, testCompressedSize, 0); + LZ4_resetStreamHC_fast(&sHC, compressionLevel); + LZ4_loadDictHC(&sHC, testInput, 32 KB); + result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); + FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); + + result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 32 KB); + FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe_usingDict() decompression failed following remote dictionary HC compression test"); + { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe_usingDict() decompression corruption"); + } } /* multiple HC compression with ext. dictionary */ { XXH64_state_t crcOrigState; |