diff options
author | Yann Collet <Cyan4973@users.noreply.github.com> | 2022-08-15 22:45:31 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-08-15 22:45:31 (GMT) |
commit | 5ff839680134437dbf4678f3d0c7b371d84f4964 (patch) | |
tree | 939d919c3903b42ed637542a4799fb3f4fa8b5fc /programs | |
parent | 416bc96faca629abcef42e56ecd2e20d26b79934 (diff) | |
parent | cfd6ab32522280079c2e6d3ea995f172b9ae0312 (diff) | |
download | lz4-5ff839680134437dbf4678f3d0c7b371d84f4964.zip lz4-5ff839680134437dbf4678f3d0c7b371d84f4964.tar.gz lz4-5ff839680134437dbf4678f3d0c7b371d84f4964.tar.bz2 |
stage v1.9.4
Diffstat (limited to 'programs')
-rw-r--r-- | programs/Makefile | 69 | ||||
-rw-r--r-- | programs/bench.c | 441 | ||||
-rw-r--r-- | programs/bench.h | 29 | ||||
-rw-r--r-- | programs/datagen.c | 2 | ||||
-rw-r--r-- | programs/datagen.h | 2 | ||||
-rw-r--r-- | programs/lz4-exe.rc.in | 3 | ||||
-rw-r--r-- | programs/lz4.1 | 16 | ||||
-rw-r--r-- | programs/lz4.1.md | 14 | ||||
-rw-r--r-- | programs/lz4cli.c | 104 | ||||
-rw-r--r-- | programs/lz4io.c | 389 | ||||
-rw-r--r-- | programs/lz4io.h | 4 | ||||
-rw-r--r-- | programs/platform.h | 4 | ||||
-rw-r--r-- | programs/util.h | 93 |
13 files changed, 708 insertions, 462 deletions
diff --git a/programs/Makefile b/programs/Makefile index c1053f6..ace0d03 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -1,6 +1,6 @@ # ########################################################################## # LZ4 programs - Makefile -# Copyright (C) Yann Collet 2011-2017 +# Copyright (C) Yann Collet 2011-2020 # # This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets # @@ -28,13 +28,14 @@ # lz4c : CLU, supporting also legacy lz4demo arguments # lz4c32: Same as lz4c, but forced to compile in 32-bits mode # ########################################################################## +SED = sed # Version numbers LZ4DIR := ../lib LIBVER_SRC := $(LZ4DIR)/lz4.h -LIBVER_MAJOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` -LIBVER_MINOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` -LIBVER_PATCH_SCRIPT:=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_MAJOR_SCRIPT:=`$(SED) -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_MINOR_SCRIPT:=`$(SED) -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_PATCH_SCRIPT:=`$(SED) -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT) LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT)) LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) @@ -51,16 +52,26 @@ DEBUGFLAGS= -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \ -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \ -Wpointer-arith -Wstrict-aliasing=1 CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) + +include ../Makefile.inc + +OS_VERSION ?= $(UNAME) -r +ifeq ($(TARGET_OS)$(shell $(OS_VERSION)),SunOS5.10) +LDFLAGS += -lrt +endif + FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) LZ4_VERSION=$(LIBVER) MD2ROFF = ronn MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="lz4 $(LZ4_VERSION)" -include ../Makefile.inc default: lz4-release +# silent mode by default; verbose can be triggered by V=1 or VERBOSE=1 +$(V)$(VERBOSE).SILENT: + all: lz4 lz4c all32: CFLAGS+=-m32 @@ -69,7 +80,7 @@ all32: all ifeq ($(WINBASED),yes) lz4-exe.rc: lz4-exe.rc.in @echo creating executable resource - $(Q)sed -e 's|@PROGNAME@|lz4|' \ + $(SED) -e 's|@PROGNAME@|lz4|' \ -e 's|@LIBVER_MAJOR@|$(LIBVER_MAJOR)|g' \ -e 's|@LIBVER_MINOR@|$(LIBVER_MINOR)|g' \ -e 's|@LIBVER_PATCH@|$(LIBVER_PATCH)|g' \ @@ -110,7 +121,7 @@ lz4c32 : $(SRCFILES) $(CC) $(FLAGS) $^ -o $@$(EXT) lz4.1: lz4.1.md $(LIBVER_SRC) - cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@ + cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | $(SED) -n '/^\.\\\".*/!p' > $@ man: lz4.1 @@ -122,10 +133,10 @@ preview-man: clean-man man clean: ifeq ($(WINBASED),yes) - $(Q)$(RM) *.rc + $(RM) *.rc endif - @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) - @$(RM) core *.o *.test tmp* \ + $(MAKE) -C $(LZ4DIR) $@ > $(VOID) + $(RM) core *.o *.test tmp* \ lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) lz4-wlib$(EXT) \ unlz4$(EXT) lz4cat$(EXT) @echo Cleaning completed @@ -160,28 +171,28 @@ MAN1DIR ?= $(mandir)/man1 man1dir ?= $(MAN1DIR) install: lz4 - @echo Installing binaries - @$(INSTALL_DIR) $(DESTDIR)$(bindir)/ $(DESTDIR)$(man1dir)/ - @$(INSTALL_PROGRAM) lz4$(EXT) $(DESTDIR)$(bindir)/lz4$(EXT) - @$(LN_S) lz4$(EXT) $(DESTDIR)$(bindir)/lz4c$(EXT) - @$(LN_S) lz4$(EXT) $(DESTDIR)$(bindir)/lz4cat$(EXT) - @$(LN_S) lz4$(EXT) $(DESTDIR)$(bindir)/unlz4$(EXT) - @echo Installing man pages - @$(INSTALL_DATA) lz4.1 $(DESTDIR)$(man1dir)/lz4.1 - @$(LN_SF) lz4.1 $(DESTDIR)$(man1dir)/lz4c.1 - @$(LN_SF) lz4.1 $(DESTDIR)$(man1dir)/lz4cat.1 - @$(LN_SF) lz4.1 $(DESTDIR)$(man1dir)/unlz4.1 + @echo Installing binaries in $(DESTDIR)$(bindir) + $(INSTALL_DIR) $(DESTDIR)$(bindir)/ $(DESTDIR)$(man1dir)/ + $(INSTALL_PROGRAM) lz4$(EXT) $(DESTDIR)$(bindir)/lz4$(EXT) + $(LN_SF) lz4$(EXT) $(DESTDIR)$(bindir)/lz4c$(EXT) + $(LN_SF) lz4$(EXT) $(DESTDIR)$(bindir)/lz4cat$(EXT) + $(LN_SF) lz4$(EXT) $(DESTDIR)$(bindir)/unlz4$(EXT) + @echo Installing man pages in $(DESTDIR)$(man1dir) + $(INSTALL_DATA) lz4.1 $(DESTDIR)$(man1dir)/lz4.1 + $(LN_SF) lz4.1 $(DESTDIR)$(man1dir)/lz4c.1 + $(LN_SF) lz4.1 $(DESTDIR)$(man1dir)/lz4cat.1 + $(LN_SF) lz4.1 $(DESTDIR)$(man1dir)/unlz4.1 @echo lz4 installation completed uninstall: - @$(RM) $(DESTDIR)$(bindir)/lz4cat$(EXT) - @$(RM) $(DESTDIR)$(bindir)/unlz4$(EXT) - @$(RM) $(DESTDIR)$(bindir)/lz4$(EXT) - @$(RM) $(DESTDIR)$(bindir)/lz4c$(EXT) - @$(RM) $(DESTDIR)$(man1dir)/lz4.1 - @$(RM) $(DESTDIR)$(man1dir)/lz4c.1 - @$(RM) $(DESTDIR)$(man1dir)/lz4cat.1 - @$(RM) $(DESTDIR)$(man1dir)/unlz4.1 + $(RM) $(DESTDIR)$(bindir)/lz4cat$(EXT) + $(RM) $(DESTDIR)$(bindir)/unlz4$(EXT) + $(RM) $(DESTDIR)$(bindir)/lz4$(EXT) + $(RM) $(DESTDIR)$(bindir)/lz4c$(EXT) + $(RM) $(DESTDIR)$(man1dir)/lz4.1 + $(RM) $(DESTDIR)$(man1dir)/lz4c.1 + $(RM) $(DESTDIR)$(man1dir)/lz4cat.1 + $(RM) $(DESTDIR)$(man1dir)/unlz4.1 @echo lz4 programs successfully uninstalled endif diff --git a/programs/bench.c b/programs/bench.c index 3357d14..4d35ef9 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -1,6 +1,6 @@ /* bench.c - Demo program to benchmark open-source compression algorithms - Copyright (C) Yann Collet 2012-2016 + Copyright (C) Yann Collet 2012-2020 GPL v2 License @@ -51,10 +51,101 @@ #include "lz4.h" #define LZ4_HC_STATIC_LINKING_ONLY #include "lz4hc.h" +#include "lz4frame.h" /* LZ4F_decompress */ /* ************************************* -* Compression parameters and functions +* Constants +***************************************/ +#ifndef LZ4_GIT_COMMIT_STRING +# define LZ4_GIT_COMMIT_STRING "" +#else +# define LZ4_GIT_COMMIT_STRING LZ4_EXPAND_AND_QUOTE(LZ4_GIT_COMMIT) +#endif + +#define NBSECONDS 3 +#define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */ +#define TIMELOOP_NANOSEC 1*1000000000ULL /* 1 second */ +#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */ +#define COOLPERIOD_SEC 10 +#define DECOMP_MULT 1 /* test decompression DECOMP_MULT times longer than compression */ + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define LZ4_MAX_DICT_SIZE (64 KB) + +static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); + +static U32 g_compressibilityDefault = 50; + + +/* ************************************* +* console display +***************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ + +#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ + if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \ + { g_time = clock(); DISPLAY(__VA_ARGS__); \ + if (g_displayLevel>=4) fflush(stdout); } } +static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; +static clock_t g_time = 0; + + +/* ************************************* +* DEBUG and error conditions +***************************************/ +#ifndef DEBUG +# define DEBUG 0 +#endif +#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); +#define END_PROCESS(error, ...) \ +{ \ + DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAYLEVEL(1, "Error %i : ", error); \ + DISPLAYLEVEL(1, __VA_ARGS__); \ + DISPLAYLEVEL(1, "\n"); \ + exit(error); \ +} + +#define LZ4_isError(errcode) (errcode==0) + + +/* ************************************* +* Benchmark Parameters +***************************************/ +static U32 g_nbSeconds = NBSECONDS; +static size_t g_blockSize = 0; +int g_additionalParam = 0; +int g_benchSeparately = 0; +int g_decodeOnly = 0; +unsigned g_skipChecksums = 0; + +void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; } + +void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; } + +void BMK_setNbSeconds(unsigned nbSeconds) +{ + g_nbSeconds = nbSeconds; + DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbSeconds); +} + +void BMK_setBlockSize(size_t blockSize) { g_blockSize = blockSize; } + +void BMK_setBenchSeparately(int separate) { g_benchSeparately = (separate!=0); } + +void BMK_setDecodeOnlyMode(int set) { g_decodeOnly = (set!=0); } + +void BMK_skipChecksums(int skip) { g_skipChecksums = (skip!=0); } + + +/* ************************************* + * Compression state management ***************************************/ struct compressionParameters @@ -79,8 +170,8 @@ struct compressionParameters const struct compressionParameters* pThis); }; -static void LZ4_compressInitNoStream( - struct compressionParameters* pThis) +static void +LZ4_compressInitNoStream(struct compressionParameters* pThis) { pThis->LZ4_stream = NULL; pThis->LZ4_dictStream = NULL; @@ -88,8 +179,8 @@ static void LZ4_compressInitNoStream( pThis->LZ4_dictStreamHC = NULL; } -static void LZ4_compressInitStream( - struct compressionParameters* pThis) +static void +LZ4_compressInitStream(struct compressionParameters* pThis) { pThis->LZ4_stream = LZ4_createStream(); pThis->LZ4_dictStream = LZ4_createStream(); @@ -98,8 +189,8 @@ static void LZ4_compressInitStream( LZ4_loadDict(pThis->LZ4_dictStream, pThis->dictBuf, pThis->dictSize); } -static void LZ4_compressInitStreamHC( - struct compressionParameters* pThis) +static void +LZ4_compressInitStreamHC(struct compressionParameters* pThis) { pThis->LZ4_stream = NULL; pThis->LZ4_dictStream = NULL; @@ -108,83 +199,84 @@ static void LZ4_compressInitStreamHC( LZ4_loadDictHC(pThis->LZ4_dictStreamHC, pThis->dictBuf, pThis->dictSize); } -static void LZ4_compressResetNoStream( - const struct compressionParameters* pThis) +static void +LZ4_compressResetNoStream(const struct compressionParameters* pThis) { (void)pThis; } -static void LZ4_compressResetStream( - const struct compressionParameters* pThis) +static void +LZ4_compressResetStream(const struct compressionParameters* pThis) { LZ4_resetStream_fast(pThis->LZ4_stream); LZ4_attach_dictionary(pThis->LZ4_stream, pThis->LZ4_dictStream); } -static void LZ4_compressResetStreamHC( - const struct compressionParameters* pThis) +static void +LZ4_compressResetStreamHC(const struct compressionParameters* pThis) { LZ4_resetStreamHC_fast(pThis->LZ4_streamHC, pThis->cLevel); LZ4_attach_HC_dictionary(pThis->LZ4_streamHC, pThis->LZ4_dictStreamHC); } -static int LZ4_compressBlockNoStream( - const struct compressionParameters* pThis, - const char* src, char* dst, - int srcSize, int dstSize) +static int +LZ4_compressBlockNoStream(const struct compressionParameters* pThis, + const char* src, char* dst, + int srcSize, int dstSize) { int const acceleration = (pThis->cLevel < 0) ? -pThis->cLevel + 1 : 1; return LZ4_compress_fast(src, dst, srcSize, dstSize, acceleration); } -static int LZ4_compressBlockNoStreamHC( - const struct compressionParameters* pThis, - const char* src, char* dst, - int srcSize, int dstSize) +static int +LZ4_compressBlockNoStreamHC(const struct compressionParameters* pThis, + const char* src, char* dst, + int srcSize, int dstSize) { return LZ4_compress_HC(src, dst, srcSize, dstSize, pThis->cLevel); } -static int LZ4_compressBlockStream( - const struct compressionParameters* pThis, - const char* src, char* dst, - int srcSize, int dstSize) +static int +LZ4_compressBlockStream(const struct compressionParameters* pThis, + const char* src, char* dst, + int srcSize, int dstSize) { int const acceleration = (pThis->cLevel < 0) ? -pThis->cLevel + 1 : 1; return LZ4_compress_fast_continue(pThis->LZ4_stream, src, dst, srcSize, dstSize, acceleration); } -static int LZ4_compressBlockStreamHC( - const struct compressionParameters* pThis, - const char* src, char* dst, - int srcSize, int dstSize) +static int +LZ4_compressBlockStreamHC(const struct compressionParameters* pThis, + const char* src, char* dst, + int srcSize, int dstSize) { return LZ4_compress_HC_continue(pThis->LZ4_streamHC, src, dst, srcSize, dstSize); } -static void LZ4_compressCleanupNoStream( - const struct compressionParameters* pThis) +static void +LZ4_compressCleanupNoStream(const struct compressionParameters* pThis) { (void)pThis; } -static void LZ4_compressCleanupStream( - const struct compressionParameters* pThis) +static void +LZ4_compressCleanupStream(const struct compressionParameters* pThis) { LZ4_freeStream(pThis->LZ4_stream); LZ4_freeStream(pThis->LZ4_dictStream); } -static void LZ4_compressCleanupStreamHC( - const struct compressionParameters* pThis) +static void +LZ4_compressCleanupStreamHC(const struct compressionParameters* pThis) { LZ4_freeStreamHC(pThis->LZ4_streamHC); LZ4_freeStreamHC(pThis->LZ4_dictStreamHC); } -static void LZ4_buildCompressionParameters( - struct compressionParameters* pParams, - int cLevel, const char* dictBuf, int dictSize) +static void +LZ4_buildCompressionParameters(struct compressionParameters* pParams, + int cLevel, + const char* dictBuf, int dictSize) { pParams->cLevel = cLevel; pParams->dictBuf = dictBuf; @@ -215,90 +307,35 @@ static void LZ4_buildCompressionParameters( } } -#define LZ4_isError(errcode) (errcode==0) +typedef int (*DecFunction_f)(const char* src, char* dst, + int srcSize, int dstCapacity, + const char* dictStart, int dictSize); -/* ************************************* -* Constants -***************************************/ -#ifndef LZ4_GIT_COMMIT_STRING -# define LZ4_GIT_COMMIT_STRING "" -#else -# define LZ4_GIT_COMMIT_STRING LZ4_EXPAND_AND_QUOTE(LZ4_GIT_COMMIT) -#endif - -#define NBSECONDS 3 -#define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */ -#define TIMELOOP_NANOSEC 1*1000000000ULL /* 1 second */ -#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */ -#define COOLPERIOD_SEC 10 -#define DECOMP_MULT 1 /* test decompression DECOMP_MULT times longer than compression */ - -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -#define LZ4_MAX_DICT_SIZE (64 KB) - -static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); - -static U32 g_compressibilityDefault = 50; - - -/* ************************************* -* console display -***************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ - -#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ - if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \ - { g_time = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stdout); } } -static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; -static clock_t g_time = 0; - - -/* ************************************* -* Exceptions -***************************************/ -#ifndef DEBUG -# define DEBUG 0 -#endif -#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); -#define EXM_THROW(error, ...) \ -{ \ - DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ - DISPLAYLEVEL(1, "Error %i : ", error); \ - DISPLAYLEVEL(1, __VA_ARGS__); \ - DISPLAYLEVEL(1, "\n"); \ - exit(error); \ -} - +static LZ4F_dctx* g_dctx = NULL; -/* ************************************* -* Benchmark Parameters -***************************************/ -static U32 g_nbSeconds = NBSECONDS; -static size_t g_blockSize = 0; -int g_additionalParam = 0; -int g_benchSeparately = 0; - -void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; } - -void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; } - -void BMK_setNbSeconds(unsigned nbSeconds) +static int +LZ4F_decompress_binding(const char* src, char* dst, + int srcSize, int dstCapacity, + const char* dictStart, int dictSize) { - g_nbSeconds = nbSeconds; - DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbSeconds); + size_t dstSize = (size_t)dstCapacity; + size_t readSize = (size_t)srcSize; + LZ4F_decompressOptions_t dOpt = { 1, 0, 0, 0 }; + size_t decStatus; + dOpt.skipChecksums = g_skipChecksums; + decStatus = LZ4F_decompress(g_dctx, + dst, &dstSize, + src, &readSize, + &dOpt); + if ( (decStatus == 0) /* decompression successful */ + && ((int)readSize==srcSize) /* consume all input */ ) + return (int)dstSize; + /* else, error */ + return -1; + (void)dictStart; (void)dictSize; /* not compatible with dictionary yet */ } -void BMK_setBlockSize(size_t blockSize) { g_blockSize = blockSize; } - -void BMK_setBenchSeparately(int separate) { g_benchSeparately = (separate!=0); } - /* ******************************************************** * Bench functions @@ -321,24 +358,32 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, const size_t* fileSizes, U32 nbFiles, const char* dictBuf, int dictSize) { - size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; - U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; + size_t const blockSize = (g_blockSize>=32 && !g_decodeOnly ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; + U32 const maxNbBlocks = (U32)((srcSize + (blockSize-1)) / blockSize) + nbFiles; blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t)); - size_t const maxCompressedSize = LZ4_compressBound((int)srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ + size_t const maxCompressedSize = (size_t)LZ4_compressBound((int)srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ void* const compressedBuffer = malloc(maxCompressedSize); - void* const resultBuffer = malloc(srcSize); + size_t const decMultiplier = g_decodeOnly ? 255 : 1; + size_t const maxInSize = (size_t)LZ4_MAX_INPUT_SIZE / decMultiplier; + size_t const maxDecSize = srcSize < maxInSize ? srcSize * decMultiplier : LZ4_MAX_INPUT_SIZE; + void* const resultBuffer = malloc(maxDecSize); U32 nbBlocks; struct compressionParameters compP; /* checks */ if (!compressedBuffer || !resultBuffer || !blockTable) - EXM_THROW(31, "allocation error : not enough memory"); + END_PROCESS(31, "allocation error : not enough memory"); if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */ /* init */ LZ4_buildCompressionParameters(&compP, cLevel, dictBuf, dictSize); compP.initFunction(&compP); + if (g_dctx==NULL) { + LZ4F_createDecompressionContext(&g_dctx, LZ4F_VERSION); + if (g_dctx==NULL) + END_PROCESS(1, "allocation error - decompression state"); + } /* Init blockTable data */ { const char* srcPtr = (const char*)srcBuffer; @@ -351,6 +396,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, U32 const blockEnd = nbBlocks + nbBlocksforThisFile; for ( ; nbBlocks<blockEnd; nbBlocks++) { size_t const thisBlockSize = MIN(remaining, blockSize); + size_t const resMaxSize = thisBlockSize * decMultiplier; + size_t const resCapa = (thisBlockSize < maxInSize) ? resMaxSize : LZ4_MAX_INPUT_SIZE; blockTable[nbBlocks].srcPtr = srcPtr; blockTable[nbBlocks].cPtr = cPtr; blockTable[nbBlocks].resPtr = resPtr; @@ -358,29 +405,37 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, blockTable[nbBlocks].cRoom = (size_t)LZ4_compressBound((int)thisBlockSize); srcPtr += thisBlockSize; cPtr += blockTable[nbBlocks].cRoom; - resPtr += thisBlockSize; + resPtr += resCapa; remaining -= thisBlockSize; } } } - /* warmimg up memory */ + /* warming up memory */ RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1); + /* decode-only mode : copy input to @compressedBuffer */ + if (g_decodeOnly) { + U32 blockNb; + for (blockNb=0; blockNb < nbBlocks; blockNb++) { + memcpy(blockTable[blockNb].cPtr, blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize); + blockTable[blockNb].cSize = blockTable[blockNb].srcSize; + } } + /* Bench */ { U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL); U64 const crcOrig = XXH64(srcBuffer, srcSize, 0); - UTIL_time_t coolTime; + UTIL_time_t coolTime = UTIL_getTime(); U64 const maxTime = (g_nbSeconds * TIMELOOP_NANOSEC) + 100; U32 nbCompressionLoops = (U32)((5 MB) / (srcSize+1)) + 1; /* conservative initial compression speed estimate */ U32 nbDecodeLoops = (U32)((200 MB) / (srcSize+1)) + 1; /* conservative initial decode speed estimate */ U64 totalCTime=0, totalDTime=0; - U32 cCompleted=0, dCompleted=0; + U32 cCompleted=(g_decodeOnly==1), dCompleted=0; # define NB_MARKS 4 const char* const marks[NB_MARKS] = { " |", " /", " =", "\\" }; U32 markNb = 0; - size_t cSize = 0; + size_t cSize = srcSize; + size_t totalRSize = srcSize; double ratio = 0.; - coolTime = UTIL_getTime(); DISPLAYLEVEL(2, "\r%79s\r", ""); while (!cCompleted || !dCompleted) { /* overheat protection */ @@ -391,8 +446,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, } /* Compression */ - DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize); - if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */ + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)totalRSize); + if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase compressed buffer */ UTIL_sleepMilli(1); /* give processor time to other processes */ UTIL_waitForNextTick(); @@ -408,7 +463,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, &compP, blockTable[blockNb].srcPtr, blockTable[blockNb].cPtr, (int)blockTable[blockNb].srcSize, (int)blockTable[blockNb].cRoom); - if (LZ4_isError(rSize)) EXM_THROW(1, "LZ4 compression failed"); + if (LZ4_isError(rSize)) END_PROCESS(1, "LZ4 compression failed"); blockTable[blockNb].cSize = rSize; } } { U64 const clockSpan = UTIL_clockSpanNano(clockStart); @@ -423,17 +478,18 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, } totalCTime += clockSpan; cCompleted = totalCTime>maxTime; - } } - - cSize = 0; - { U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; } - cSize += !cSize; /* avoid div by 0 */ - ratio = (double)srcSize / (double)cSize; - markNb = (markNb+1) % NB_MARKS; - DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r", - marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio, - ((double)srcSize / fastestC) * 1000 ); - + } + + cSize = 0; + { U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; } + cSize += !cSize; /* avoid div by 0 */ + ratio = (double)totalRSize / (double)cSize; + markNb = (markNb+1) % NB_MARKS; + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r", + marks[markNb], displayName, + (U32)totalRSize, (U32)cSize, ratio, + ((double)totalRSize / fastestC) * 1000 ); + } (void)fastestD; (void)crcOrig; /* unused when decompression disabled */ #if 1 /* Decompression */ @@ -443,17 +499,30 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, UTIL_waitForNextTick(); if (!dCompleted) { + const DecFunction_f decFunction = g_decodeOnly ? + LZ4F_decompress_binding : LZ4_decompress_safe_usingDict; + const char* const decString = g_decodeOnly ? + "LZ4F_decompress" : "LZ4_decompress_safe_usingDict"; UTIL_time_t const clockStart = UTIL_getTime(); U32 nbLoops; + for (nbLoops=0; nbLoops < nbDecodeLoops; nbLoops++) { U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) { - int const regenSize = LZ4_decompress_safe_usingDict( + size_t const inMaxSize = (size_t)INT_MAX / decMultiplier; + size_t const resCapa = (blockTable[blockNb].srcSize < inMaxSize) ? + blockTable[blockNb].srcSize * decMultiplier : + INT_MAX; + int const regenSize = decFunction( blockTable[blockNb].cPtr, blockTable[blockNb].resPtr, - (int)blockTable[blockNb].cSize, (int)blockTable[blockNb].srcSize, + (int)blockTable[blockNb].cSize, (int)resCapa, dictBuf, dictSize); if (regenSize < 0) { - DISPLAY("LZ4_decompress_safe_usingDict() failed on block %u \n", blockNb); + DISPLAY("%s() failed on block %u of size %u \n", + decString, blockNb, (unsigned)blockTable[blockNb].srcSize); + if (g_decodeOnly) + DISPLAY("Is input using LZ4 Frame format ? \n"); + END_PROCESS(2, "error during decoding"); break; } blockTable[blockNb].resSize = (size_t)regenSize; @@ -472,14 +541,22 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, dCompleted = totalDTime > (DECOMP_MULT*maxTime); } } + if (g_decodeOnly) { + unsigned u; + totalRSize = 0; + for (u=0; u<nbBlocks; u++) totalRSize += blockTable[u].resSize; + } markNb = (markNb+1) % NB_MARKS; + ratio = (double)totalRSize / (double)cSize; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", - marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio, - ((double)srcSize / fastestC) * 1000, - ((double)srcSize / fastestD) * 1000); - - /* CRC Checking */ - { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); + marks[markNb], displayName, + (U32)totalRSize, (U32)cSize, ratio, + ((double)totalRSize / fastestC) * 1000, + ((double)totalRSize / fastestD) * 1000); + + /* CRC Checking (not possible in decode-only mode)*/ + if (!g_decodeOnly) { + U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); if (crcOrig!=crcCheck) { size_t u; DISPLAY("\n!!! WARNING !!! %17s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck); @@ -594,21 +671,21 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize, continue; } f = fopen(fileNamesTable[n], "rb"); - if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]); + if (f==NULL) END_PROCESS(10, "impossible to open file %s", fileNamesTable[n]); DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]); if (fileSize > bufferSize-pos) { /* buffer too small - stop after this file */ fileSize = bufferSize-pos; nbFiles=n; } { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); - if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]); + if (readSize != (size_t)fileSize) END_PROCESS(11, "could not read %s", fileNamesTable[n]); pos += readSize; } fileSizes[n] = (size_t)fileSize; totalSize += (size_t)fileSize; fclose(f); } - if (totalSize == 0) EXM_THROW(12, "no data to bench"); + if (totalSize == 0) END_PROCESS(12, "no data to bench"); } static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, @@ -621,11 +698,11 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); char mfName[20] = {0}; - if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes"); + if (!fileSizes) END_PROCESS(12, "not enough memory for fileSizes"); /* Memory allocation & restrictions */ benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; - if (benchedSize==0) EXM_THROW(12, "not enough memory"); + if (benchedSize==0) END_PROCESS(12, "not enough memory"); if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad; if (benchedSize > LZ4_MAX_INPUT_SIZE) { benchedSize = LZ4_MAX_INPUT_SIZE; @@ -635,7 +712,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20)); } srcBuffer = malloc(benchedSize + !benchedSize); /* avoid alloc of zero */ - if (!srcBuffer) EXM_THROW(12, "not enough memory"); + if (!srcBuffer) END_PROCESS(12, "not enough memory"); /* Load input buffer */ BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles); @@ -663,7 +740,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility void* const srcBuffer = malloc(benchedSize); /* Memory allocation */ - if (!srcBuffer) EXM_THROW(21, "not enough memory"); + if (!srcBuffer) END_PROCESS(21, "not enough memory"); /* Fill input buffer */ RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0); @@ -677,7 +754,8 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility } -int BMK_benchFilesSeparately(const char** fileNamesTable, unsigned nbFiles, +static int +BMK_benchFilesSeparately(const char** fileNamesTable, unsigned nbFiles, int cLevel, int cLevelLast, const char* dictBuf, int dictSize) { @@ -685,7 +763,6 @@ int BMK_benchFilesSeparately(const char** fileNamesTable, unsigned nbFiles, if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX; if (cLevelLast > LZ4HC_CLEVEL_MAX) cLevelLast = LZ4HC_CLEVEL_MAX; if (cLevelLast < cLevel) cLevelLast = cLevel; - if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); for (fileNb=0; fileNb<nbFiles; fileNb++) BMK_benchFileTable(fileNamesTable+fileNb, 1, cLevel, cLevelLast, dictBuf, dictSize); @@ -700,45 +777,59 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, { double const compressibility = (double)g_compressibilityDefault / 100; char* dictBuf = NULL; - int dictSize = 0; + size_t dictSize = 0; if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX; + if (g_decodeOnly) { + DISPLAYLEVEL(2, "Benchmark Decompression of LZ4 Frame "); + if (g_skipChecksums) { + DISPLAYLEVEL(2, "_without_ checksum even when present \n"); + } else { + DISPLAYLEVEL(2, "+ Checksum when present \n"); + } + cLevelLast = cLevel; + } if (cLevelLast > LZ4HC_CLEVEL_MAX) cLevelLast = LZ4HC_CLEVEL_MAX; if (cLevelLast < cLevel) cLevelLast = cLevel; - if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); + if (cLevelLast > cLevel) + DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); if (dictFileName) { FILE* dictFile = NULL; - U64 dictFileSize = UTIL_getFileSize(dictFileName); - if (!dictFileSize) EXM_THROW(25, "Dictionary error : could not stat dictionary file"); + U64 const dictFileSize = UTIL_getFileSize(dictFileName); + if (!dictFileSize) + END_PROCESS(25, "Dictionary error : could not stat dictionary file"); + if (g_decodeOnly) + END_PROCESS(26, "Error : LZ4 Frame decoder mode not compatible with dictionary yet"); dictFile = fopen(dictFileName, "rb"); - if (!dictFile) EXM_THROW(25, "Dictionary error : could not open dictionary file"); + if (!dictFile) + END_PROCESS(25, "Dictionary error : could not open dictionary file"); if (dictFileSize > LZ4_MAX_DICT_SIZE) { dictSize = LZ4_MAX_DICT_SIZE; - if (UTIL_fseek(dictFile, dictFileSize - dictSize, SEEK_SET)) - EXM_THROW(25, "Dictionary error : could not seek dictionary file"); + if (UTIL_fseek(dictFile, (long)(dictFileSize - dictSize), SEEK_SET)) + END_PROCESS(25, "Dictionary error : could not seek dictionary file"); } else { - dictSize = (int)dictFileSize; + dictSize = (size_t)dictFileSize; } - dictBuf = (char *)malloc(dictSize); - if (!dictBuf) EXM_THROW(25, "Allocation error : not enough memory"); + dictBuf = (char*)malloc(dictSize); + if (!dictBuf) END_PROCESS(25, "Allocation error : not enough memory"); - if (fread(dictBuf, 1, dictSize, dictFile) != (size_t)dictSize) - EXM_THROW(25, "Dictionary error : could not read dictionary file"); + if (fread(dictBuf, 1, dictSize, dictFile) != dictSize) + END_PROCESS(25, "Dictionary error : could not read dictionary file"); fclose(dictFile); } if (nbFiles == 0) - BMK_syntheticTest(cLevel, cLevelLast, compressibility, dictBuf, dictSize); + BMK_syntheticTest(cLevel, cLevelLast, compressibility, dictBuf, (int)dictSize); else { if (g_benchSeparately) - BMK_benchFilesSeparately(fileNamesTable, nbFiles, cLevel, cLevelLast, dictBuf, dictSize); + BMK_benchFilesSeparately(fileNamesTable, nbFiles, cLevel, cLevelLast, dictBuf, (int)dictSize); else - BMK_benchFileTable(fileNamesTable, nbFiles, cLevel, cLevelLast, dictBuf, dictSize); + BMK_benchFileTable(fileNamesTable, nbFiles, cLevel, cLevelLast, dictBuf, (int)dictSize); } free(dictBuf); diff --git a/programs/bench.h b/programs/bench.h index 22ebf60..1d81a99 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -1,6 +1,6 @@ /* bench.h - Demo program to benchmark open-source compression algorithm - Copyright (C) Yann Collet 2012-2016 + Copyright (C) Yann Collet 2012-2020 This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -25,15 +25,30 @@ #include <stddef.h> +/* BMK_benchFiles() : + * Benchmark all files provided through array @fileNamesTable. + * All files must be valid, otherwise benchmark fails. + * Roundtrip measurements are done for each file individually, but + * unless BMK_setBenchSeparately() is set, all results are agglomerated. + * The method benchmarks all compression levels from @cLevelStart to @cLevelLast, + * both inclusive, providing one result per compression level. + * If @cLevelLast <= @cLevelStart, BMK_benchFiles() benchmarks @cLevelStart only. + * @dictFileName is optional, it's possible to provide NULL. + * When provided, compression and decompression use the specified file as dictionary. + * Only one dictionary can be provided, in which case it's applied to all benchmarked files. +**/ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - int cLevel, int cLevelLast, + int cLevelStart, int cLevelLast, const char* dictFileName); /* Set Parameters */ -void BMK_setNbSeconds(unsigned nbLoops); -void BMK_setBlockSize(size_t blockSize); -void BMK_setAdditionalParam(int additionalParam); -void BMK_setNotificationLevel(unsigned level); -void BMK_setBenchSeparately(int separate); +void BMK_setNbSeconds(unsigned nbSeconds); /* minimum benchmark duration, in seconds, for both compression and decompression */ +void BMK_setBlockSize(size_t blockSize); /* Internally cut input file(s) into independent blocks of specified size */ +void BMK_setNotificationLevel(unsigned level); /* Influence verbosity level */ +void BMK_setBenchSeparately(int separate); /* When providing multiple files, output one result per file */ +void BMK_setDecodeOnlyMode(int set); /* v1.9.4+: set benchmark mode to decode only */ +void BMK_skipChecksums(int skip); /* v1.9.4+: only useful for DecodeOnlyMode; do not calculate checksum when present, to save CPU time */ + +void BMK_setAdditionalParam(int additionalParam); /* hidden param, influence output format, for python parsing */ #endif /* BENCH_H_125623623633 */ diff --git a/programs/datagen.c b/programs/datagen.c index 24a2da2..f448640 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -1,6 +1,6 @@ /* datagen.c - compressible data generator test tool - Copyright (C) Yann Collet 2012-2016 + Copyright (C) Yann Collet 2012-2020 GPL v2 License diff --git a/programs/datagen.h b/programs/datagen.h index 91c5b02..c20c9c7 100644 --- a/programs/datagen.h +++ b/programs/datagen.h @@ -1,6 +1,6 @@ /* datagen.h - compressible data generator header - Copyright (C) Yann Collet 2012-2016 + Copyright (C) Yann Collet 2012-2020 GPL v2 License diff --git a/programs/lz4-exe.rc.in b/programs/lz4-exe.rc.in index 7b81030..bcf4d7d 100644 --- a/programs/lz4-exe.rc.in +++ b/programs/lz4-exe.rc.in @@ -13,7 +13,7 @@ FILETYPE 1 VALUE "FileDescription", "Extremely fast compression" VALUE "FileVersion", "@LIBVER_MAJOR@.@LIBVER_MINOR@.@LIBVER_PATCH@.0" VALUE "InternalName", "@PROGNAME@" - VALUE "LegalCopyright", "Copyright (C) 2013-2016, Yann Collet" + VALUE "LegalCopyright", "Copyright (C) 2013-2020, Yann Collet" VALUE "OriginalFilename", "@PROGNAME@.@EXT@" VALUE "ProductName", "LZ4" VALUE "ProductVersion", "@LIBVER_MAJOR@.@LIBVER_MINOR@.@LIBVER_PATCH@.0" @@ -24,4 +24,3 @@ FILETYPE 1 VALUE "Translation", 0x0409, 1200 } } - diff --git a/programs/lz4.1 b/programs/lz4.1 index d758ed5..7cb98d6 100644 --- a/programs/lz4.1 +++ b/programs/lz4.1 @@ -1,5 +1,5 @@ . -.TH "LZ4" "1" "July 2019" "lz4 1.9.2" "User Commands" +.TH "LZ4" "1" "August 2022" "lz4 v1.9.4" "User Commands" . .SH "NAME" \fBlz4\fR \- lz4, unlz4, lz4cat \- Compress or decompress \.lz4 files @@ -17,7 +17,7 @@ When writing scripts that need to decompress files, it is recommended to always use the name \fBlz4\fR with appropriate arguments (\fBlz4 \-d\fR or \fBlz4 \-dc\fR) instead of the names \fBunlz4\fR and \fBlz4cat\fR\. . .SH "DESCRIPTION" -\fBlz4\fR is an extremely fast lossless compression algorithm, based on \fBbyte\-aligned LZ77\fR family of compression scheme\. \fBlz4\fR offers compression speeds of 400 MB/s per core, linearly scalable with multi\-core CPUs\. It features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limit on multi\-core systems\. The native file format is the \fB\.lz4\fR format\. +\fBlz4\fR is an extremely fast lossless compression algorithm, based on \fBbyte\-aligned LZ77\fR family of compression scheme\. \fBlz4\fR offers compression speeds > 500 MB/s per core, linearly scalable with multi\-core CPUs\. It features an extremely fast decoder, offering speed in multiple GB/s per core, typically reaching RAM speed limit on multi\-core systems\. The native file format is the \fB\.lz4\fR format\. . .SS "Difference between lz4 and gzip" \fBlz4\fR supports a command line syntax similar \fIbut not identical\fR to \fBgzip(1)\fR\. Differences are : @@ -32,7 +32,7 @@ When writing scripts that need to decompress files, it is recommended to always \fBlz4 file\.lz4\fR will default to decompression (use \fB\-z\fR to force compression) . .IP "\(bu" 4 -\fBlz4\fR preserves original files +\fBlz4\fR preserves original files (see \fB\-\-rm\fR to erase source file on completion) . .IP "\(bu" 4 \fBlz4\fR shows real\-time notification statistics during compression or decompression of a single file (use \fB\-q\fR to silence them) @@ -121,7 +121,7 @@ Switch to ultra\-fast compression levels\. The higher the value, the faster the . .TP \fB\-\-best\fR -Set highest compression level\. Same as -12\. +Set highest compression level\. Same as \-12\. . .TP \fB\-\-favor\-decSpeed\fR @@ -169,10 +169,18 @@ Produce independent blocks (default) Blocks depend on predecessors (improves compression ratio, more noticeable on small blocks) . .TP +\fB\-BX\fR +Generate block checksums (default:disabled) +. +.TP \fB\-\-[no\-]frame\-crc\fR Select frame checksum (default:enabled) . .TP +\fB\-\-no\-crc\fR +Disable both frame and block checksums +. +.TP \fB\-\-[no\-]content\-size\fR Header includes original size (default:not present) . diff --git a/programs/lz4.1.md b/programs/lz4.1.md index 56c0053..06c06cf 100644 --- a/programs/lz4.1.md +++ b/programs/lz4.1.md @@ -20,9 +20,9 @@ DESCRIPTION `lz4` is an extremely fast lossless compression algorithm, based on **byte-aligned LZ77** family of compression scheme. -`lz4` offers compression speeds of 400 MB/s per core, linearly scalable with -multi-core CPUs. -It features an extremely fast decoder, with speed in multiple GB/s per core, +`lz4` offers compression speeds > 500 MB/s per core, +linearly scalable with multi-core CPUs. +It features an extremely fast decoder, offering speed in multiple GB/s per core, typically reaching RAM speed limit on multi-core systems. The native file format is the `.lz4` format. @@ -34,7 +34,7 @@ Differences are : * `lz4` compresses a single file by default (see `-m` for multiple files) * `lz4 file1 file2` means : compress file1 _into_ file2 * `lz4 file.lz4` will default to decompression (use `-z` to force compression) - * `lz4` preserves original files + * `lz4` preserves original files (see `--rm` to erase source file on completion) * `lz4` shows real-time notification statistics during compression or decompression of a single file (use `-q` to silence them) @@ -185,9 +185,15 @@ only the latest one will be applied. * `-BD`: Blocks depend on predecessors (improves compression ratio, more noticeable on small blocks) +* `-BX`: + Generate block checksums (default:disabled) + * `--[no-]frame-crc`: Select frame checksum (default:enabled) +* `--no-crc`: + Disable both frame and block checksums + * `--[no-]content-size`: Header includes original size (default:not present)<br/> Note : this option can only be activated when the original size can be diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 523b8a8..8c3f9fd 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -1,6 +1,6 @@ /* LZ4cli - LZ4 Command Line Interface - Copyright (C) Yann Collet 2011-2016 + Copyright (C) Yann Collet 2011-2020 GPL v2 License @@ -186,7 +186,7 @@ static int usage_longhelp(const char* exeName) DISPLAY( "\n"); DISPLAY( "Compression levels : \n"); DISPLAY( "---------------------\n"); - DISPLAY( "-0 ... -2 => Fast compression, all identicals\n"); + DISPLAY( "-0 ... -2 => Fast compression, all identical\n"); DISPLAY( "-3 ... -%d => High compression; higher number == more compression but slower\n", LZ4HC_CLEVEL_MAX); DISPLAY( "\n"); DISPLAY( "stdin, stdout and the console : \n"); @@ -314,6 +314,7 @@ int main(int argc, const char** argv) cLevelLast=-10000, legacy_format=0, forceStdout=0, + forceOverwrite=0, main_pause=0, multiple_inputs=0, all_arguments_are_files=0, @@ -330,9 +331,8 @@ int main(int argc, const char** argv) const char extension[] = LZ4_EXTENSION; size_t blockSize = LZ4IO_setBlockSizeID(prefs, LZ4_BLOCKSIZEID_DEFAULT); const char* const exeName = lastNameFromPath(argv[0]); -#ifdef UTIL_HAS_CREATEFILELIST - const char** extendedFileList = NULL; char* fileNamesBuf = NULL; +#ifdef UTIL_HAS_CREATEFILELIST unsigned fileNamesNb, recursive=0; #endif @@ -377,16 +377,21 @@ int main(int argc, const char** argv) if (argument[1]=='-') { if (!strcmp(argument, "--")) { all_arguments_are_files = 1; continue; } if (!strcmp(argument, "--compress")) { mode = om_compress; continue; } - if ((!strcmp(argument, "--decompress")) - || (!strcmp(argument, "--uncompress"))) { mode = om_decompress; continue; } + if ( (!strcmp(argument, "--decompress")) + || (!strcmp(argument, "--uncompress"))) { + if (mode != om_bench) mode = om_decompress; + BMK_setDecodeOnlyMode(1); + continue; + } if (!strcmp(argument, "--multiple")) { multiple_inputs = 1; continue; } if (!strcmp(argument, "--test")) { mode = om_test; continue; } if (!strcmp(argument, "--force")) { LZ4IO_setOverwrite(prefs, 1); continue; } if (!strcmp(argument, "--no-force")) { LZ4IO_setOverwrite(prefs, 0); continue; } if ((!strcmp(argument, "--stdout")) || (!strcmp(argument, "--to-stdout"))) { forceStdout=1; output_filename=stdoutmark; continue; } - if (!strcmp(argument, "--frame-crc")) { LZ4IO_setStreamChecksumMode(prefs, 1); continue; } - if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(prefs, 0); continue; } + if (!strcmp(argument, "--frame-crc")) { LZ4IO_setStreamChecksumMode(prefs, 1); BMK_skipChecksums(0); continue; } + if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(prefs, 0); BMK_skipChecksums(1); continue; } + if (!strcmp(argument, "--no-crc")) { LZ4IO_setStreamChecksumMode(prefs, 0); LZ4IO_setBlockChecksumMode(prefs, 0); BMK_skipChecksums(1); continue; } if (!strcmp(argument, "--content-size")) { LZ4IO_setContentSize(prefs, 1); continue; } if (!strcmp(argument, "--no-content-size")) { LZ4IO_setContentSize(prefs, 0); continue; } if (!strcmp(argument, "--list")) { mode = om_list; continue; } @@ -478,7 +483,10 @@ int main(int argc, const char** argv) case 'l': legacy_format = 1; blockSize = 8 MB; break; /* Decoding */ - case 'd': mode = om_decompress; break; + case 'd': + if (mode != om_bench) mode = om_decompress; + BMK_setDecodeOnlyMode(1); + break; /* Force stdout, even if stdout==console */ case 'c': @@ -491,7 +499,7 @@ int main(int argc, const char** argv) case 't': mode = om_test; break; /* Overwrite */ - case 'f': LZ4IO_setOverwrite(prefs, 1); break; + case 'f': forceOverwrite=1; LZ4IO_setOverwrite(prefs, 1); break; /* Verbose mode */ case 'v': displayLevel++; break; @@ -581,20 +589,24 @@ int main(int argc, const char** argv) } /* Store in *inFileNames[] if -m is used. */ - if (multiple_inputs) { inFileNames[ifnIdx++]=argument; continue; } + if (multiple_inputs) { inFileNames[ifnIdx++] = argument; continue; } - /* Store first non-option arg in input_filename to preserve original cli logic. */ - if (!input_filename) { input_filename=argument; continue; } + /* original cli logic : lz4 input output */ + /* First non-option arg is input_filename. */ + if (!input_filename) { input_filename = argument; continue; } - /* Second non-option arg in output_filename to preserve original cli logic. */ + /* Second non-option arg is output_filename */ if (!output_filename) { - output_filename=argument; + output_filename = argument; if (!strcmp (output_filename, nullOutput)) output_filename = nulmark; continue; } - /* 3rd non-option arg should not exist */ - DISPLAYLEVEL(1, "Warning : %s won't be used ! Do you want multiple input files (-m) ? \n", argument); + /* 3rd+ non-option arg should not exist */ + DISPLAYLEVEL(1, "%s : %s won't be used ! Do you want multiple input files (-m) ? \n", + forceOverwrite ? "Warning" : "Error", + argument); + if (!forceOverwrite) exit(1); } DISPLAYLEVEL(3, WELCOME_MESSAGE); @@ -617,7 +629,7 @@ int main(int argc, const char** argv) input_filename = inFileNames[0]; #ifdef UTIL_HAS_CREATEFILELIST if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */ - extendedFileList = UTIL_createFileList(inFileNames, ifnIdx, &fileNamesBuf, &fileNamesNb); + const char** extendedFileList = UTIL_createFileList(inFileNames, ifnIdx, &fileNamesBuf, &fileNamesNb); if (extendedFileList) { unsigned u; for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, extendedFileList[u]); @@ -649,27 +661,19 @@ int main(int argc, const char** argv) mode = om_decompress; /* defer to decompress */ } - /* compress or decompress */ + /* No input provided => use stdin */ if (!input_filename) input_filename = stdinmark; - /* Check if input is defined as console; trigger an error in this case */ + + /* Refuse to use the console as input */ if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) { DISPLAYLEVEL(1, "refusing to read from a console\n"); exit(1); } + if (!strcmp(input_filename, stdinmark)) { /* if input==stdin and no output defined, stdout becomes default output */ if (!output_filename) output_filename = stdoutmark; } - else{ -#ifdef UTIL_HAS_CREATEFILELIST - if (!recursive && !UTIL_isRegFile(input_filename)) { -#else - if (!UTIL_isRegFile(input_filename)) { -#endif - DISPLAYLEVEL(1, "%s: is not a regular file \n", input_filename); - exit(1); - } - } /* No output filename ==> try to select one automatically (when possible) */ while ((!output_filename) && (multiple_inputs==0)) { @@ -679,7 +683,7 @@ int main(int argc, const char** argv) * To ensure `stdout` is explicitly selected, use `-c` command flag. * Conversely, to ensure output will not become `stdout`, use `-m` command flag */ DISPLAYLEVEL(1, "Warning : using stdout as default output. Do not rely on this behavior: use explicit `-c` instead ! \n"); - output_filename=stdoutmark; + output_filename = stdoutmark; break; } if (mode == om_auto) { /* auto-determine compression or decompression, based on file extension */ @@ -695,7 +699,7 @@ int main(int argc, const char** argv) DISPLAYLEVEL(2, "Compressed filename will be : %s \n", output_filename); break; } - if (mode == om_decompress) {/* decompression to file (automatic name will work only if input filename has correct format extension) */ + if (mode == om_decompress) {/* decompress to file (automatic output name only works if input filename has correct format extension) */ size_t outl; size_t const inl = strlen(input_filename); dynNameSpace = (char*)calloc(1,inl+1); @@ -704,32 +708,27 @@ int main(int argc, const char** argv) outl = inl; if (inl>4) while ((outl >= inl-4) && (input_filename[outl] == extension[outl-inl+4])) dynNameSpace[outl--]=0; - if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); badusage(exeName); } + if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename \n"); badusage(exeName); } output_filename = dynNameSpace; DISPLAYLEVEL(2, "Decoding file %s \n", output_filename); } break; } - if (mode == om_list){ - /* Exit if trying to read from stdin as this isn't supported in this mode */ - if(!strcmp(input_filename, stdinmark)){ - DISPLAYLEVEL(1, "refusing to read from standard input in --list mode\n"); - exit(1); - } - if(!multiple_inputs){ - inFileNames[ifnIdx++] = input_filename; - } - } - else{ - if (multiple_inputs==0) assert(output_filename); + if (mode == om_list) { + if (!multiple_inputs) inFileNames[ifnIdx++] = input_filename; + } else { + if (!multiple_inputs) assert(output_filename != NULL); } /* when multiple_inputs==1, output_filename may simply be useless, * however, output_filename must be !NULL for next strcmp() tests */ if (!output_filename) output_filename = "*\\dummy^!//"; /* Check if output is defined as console; trigger an error in this case */ - if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) { + if ( !strcmp(output_filename,stdoutmark) + && mode != om_list + && IS_CONSOLE(stdout) + && !forceStdout) { DISPLAYLEVEL(1, "refusing to write to console without -c \n"); exit(1); } @@ -747,8 +746,10 @@ int main(int argc, const char** argv) if (ifnIdx == 0) multiple_inputs = 0; if (mode == om_decompress) { if (multiple_inputs) { - const char* const dec_extension = !strcmp(output_filename,stdoutmark) ? stdoutmark : LZ4_EXTENSION; - assert(ifnIdx <= INT_MAX); + const char* dec_extension = LZ4_EXTENSION; + if (!strcmp(output_filename, stdoutmark)) dec_extension = stdoutmark; + if (!strcmp(output_filename, nulmark)) dec_extension = nulmark; + assert(ifnIdx < INT_MAX); operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, (int)ifnIdx, dec_extension, prefs); } else { operationResult = DEFAULT_DECOMPRESSOR(input_filename, output_filename, prefs); @@ -776,12 +777,7 @@ int main(int argc, const char** argv) _cleanup: if (main_pause) waitEnter(); free(dynNameSpace); -#ifdef UTIL_HAS_CREATEFILELIST - if (extendedFileList) { - UTIL_freeFileList(extendedFileList, fileNamesBuf); - inFileNames = NULL; - } -#endif + free(fileNamesBuf); LZ4IO_freePreferences(prefs); free((void*)inFileNames); return operationResult; diff --git a/programs/lz4io.c b/programs/lz4io.c index a274798..8b70b91 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -1,6 +1,6 @@ /* LZ4io.c - LZ4 File/Stream Interface - Copyright (C) Yann Collet 2011-2017 + Copyright (C) Yann Collet 2011-2020 GPL v2 License @@ -103,29 +103,9 @@ static int g_displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result } } static const clock_t refreshRate = CLOCKS_PER_SEC / 6; static clock_t g_time = 0; -#define LZ4IO_STATIC_ASSERT(c) { enum { LZ4IO_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */ - -/************************************** -* Local Parameters -**************************************/ +#define LZ4IO_STATIC_ASSERT(c) { enum { LZ4IO_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */ -struct LZ4IO_prefs_s { - int passThrough; - int overwrite; - int testMode; - int blockSizeId; - size_t blockSize; - int blockChecksum; - int streamChecksum; - int blockIndependence; - int sparseFileSupport; - int contentSizeFlag; - int useDictionary; - unsigned favorDecSpeed; - const char* dictionaryFilename; - int removeSrcFile; -}; /************************************** * Exceptions @@ -134,7 +114,7 @@ struct LZ4IO_prefs_s { # define DEBUG 0 #endif #define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); -#define EXM_THROW(error, ...) \ +#define END_PROCESS(error, ...) \ { \ DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ DISPLAYLEVEL(1, "Error %i : ", error); \ @@ -144,23 +124,31 @@ struct LZ4IO_prefs_s { } -/************************************** -* Version modifiers -**************************************/ -#define EXTENDED_ARGUMENTS -#define EXTENDED_HELP -#define EXTENDED_FORMAT -#define DEFAULT_DECOMPRESSOR LZ4IO_decompressLZ4F - - /* ************************************************** */ /* ****************** Parameters ******************** */ /* ************************************************** */ +struct LZ4IO_prefs_s { + int passThrough; + int overwrite; + int testMode; + int blockSizeId; + size_t blockSize; + int blockChecksum; + int streamChecksum; + int blockIndependence; + int sparseFileSupport; + int contentSizeFlag; + int useDictionary; + unsigned favorDecSpeed; + const char* dictionaryFilename; + int removeSrcFile; +}; + LZ4IO_prefs_t* LZ4IO_defaultPreferences(void) { LZ4IO_prefs_t* const ret = (LZ4IO_prefs_t*)malloc(sizeof(*ret)); - if (!ret) EXM_THROW(21, "Allocation error : not enough memory"); + if (!ret) END_PROCESS(21, "Allocation error : not enough memory"); ret->passThrough = 0; ret->overwrite = 1; ret->testMode = 0; @@ -298,6 +286,26 @@ void LZ4IO_setRemoveSrcFile(LZ4IO_prefs_t* const prefs, unsigned flag) /* ************************************************************************ ** +** ********************** String functions ********************* ** +** ************************************************************************ */ + +static int LZ4IO_isDevNull(const char* s) +{ + return UTIL_sameString(s, nulmark); +} + +static int LZ4IO_isStdin(const char* s) +{ + return UTIL_sameString(s, stdinmark); +} + +static int LZ4IO_isStdout(const char* s) +{ + return UTIL_sameString(s, stdoutmark); +} + + +/* ************************************************************************ ** ** ********************** LZ4 File / Pipe compression ********************* ** ** ************************************************************************ */ @@ -313,13 +321,13 @@ static FILE* LZ4IO_openSrcFile(const char* srcFileName) { FILE* f; - if (!strcmp (srcFileName, stdinmark)) { - DISPLAYLEVEL(4,"Using stdin for input\n"); + if (LZ4IO_isStdin(srcFileName)) { + DISPLAYLEVEL(4,"Using stdin for input \n"); f = stdin; SET_BINARY_MODE(stdin); } else { f = fopen(srcFileName, "rb"); - if ( f==NULL ) DISPLAYLEVEL(1, "%s: %s \n", srcFileName, strerror(errno)); + if (f==NULL) DISPLAYLEVEL(1, "%s: %s \n", srcFileName, strerror(errno)); } return f; @@ -334,7 +342,7 @@ static FILE* LZ4IO_openDstFile(const char* dstFileName, const LZ4IO_prefs_t* con FILE* f; assert(dstFileName != NULL); - if (!strcmp (dstFileName, stdoutmark)) { + if (LZ4IO_isStdout(dstFileName)) { DISPLAYLEVEL(4, "Using stdout for output \n"); f = stdout; SET_BINARY_MODE(stdout); @@ -343,7 +351,8 @@ static FILE* LZ4IO_openDstFile(const char* dstFileName, const LZ4IO_prefs_t* con " to force-enable it, add --sparse command \n"); } } else { - if (!prefs->overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */ + if (!prefs->overwrite && !LZ4IO_isDevNull(dstFileName)) { + /* Check if destination file already exists */ FILE* const testf = fopen( dstFileName, "rb" ); if (testf != NULL) { /* dest exists, prompt for overwrite authorization */ fclose(testf); @@ -351,7 +360,7 @@ static FILE* LZ4IO_openDstFile(const char* dstFileName, const LZ4IO_prefs_t* con DISPLAY("%s already exists; not overwritten \n", dstFileName); return NULL; } - DISPLAY("%s already exists; do you wish to overwrite (y/N) ? ", dstFileName); + DISPLAY("%s already exists; do you want to overwrite (y/N) ? ", dstFileName); { int ch = getchar(); if ((ch!='Y') && (ch!='y')) { DISPLAY(" not overwritten \n"); @@ -377,7 +386,11 @@ static FILE* LZ4IO_openDstFile(const char* dstFileName, const LZ4IO_prefs_t* con * Legacy Compression ***************************************/ -/* unoptimized version; solves endianess & alignment issues */ +/* Size in bytes of a legacy block header in little-endian format */ +#define LZ4IO_LEGACY_BLOCK_HEADER_SIZE 4 +#define LZ4IO_LEGACY_BLOCK_SIZE_MAX (8 MB) + +/* unoptimized version; solves endianness & alignment issues */ static void LZ4IO_writeLE32 (void* p, unsigned value32) { unsigned char* const dstPtr = (unsigned char*)p; @@ -413,24 +426,24 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output /* Init */ clock_t const clockStart = clock(); if (finput == NULL) - EXM_THROW(20, "%s : open file error ", input_filename); + END_PROCESS(20, "%s : open file error ", input_filename); foutput = LZ4IO_openDstFile(output_filename, prefs); if (foutput == NULL) { fclose(finput); - EXM_THROW(20, "%s : open file error ", input_filename); + END_PROCESS(20, "%s : open file error ", input_filename); } /* Allocate Memory */ in_buff = (char*)malloc(LEGACY_BLOCKSIZE); out_buff = (char*)malloc((size_t)outBuffSize + 4); if (!in_buff || !out_buff) - EXM_THROW(21, "Allocation error : not enough memory"); + END_PROCESS(21, "Allocation error : not enough memory"); /* Write Archive Header */ LZ4IO_writeLE32(out_buff, LEGACY_MAGICNUMBER); if (fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput) != MAGICNUMBER_SIZE) - EXM_THROW(22, "Write error : cannot write header"); + END_PROCESS(22, "Write error : cannot write header"); /* Main Loop */ while (1) { @@ -445,7 +458,7 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output outSize = compressionFunction(in_buff, out_buff+4, (int)inSize, outBuffSize, compressionlevel); assert(outSize >= 0); compressedfilesize += (unsigned long long)outSize+4; - DISPLAYUPDATE(2, "\rRead : %i MB ==> %.2f%% ", + DISPLAYUPDATE(2, "\rRead : %i MiB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); /* Write Block */ @@ -453,19 +466,19 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output assert(outSize < outBuffSize); LZ4IO_writeLE32(out_buff, (unsigned)outSize); if (fwrite(out_buff, 1, (size_t)outSize+4, foutput) != (size_t)(outSize+4)) { - EXM_THROW(24, "Write error : cannot write compressed block"); + END_PROCESS(24, "Write error : cannot write compressed block"); } } - if (ferror(finput)) EXM_THROW(25, "Error while reading %s ", input_filename); + if (ferror(finput)) END_PROCESS(24, "Error while reading %s ", input_filename); /* Status */ clockEnd = clock(); - if (clockEnd==clockStart) clockEnd+=1; /* avoid division by zero (speed) */ + clockEnd += (clockEnd==clockStart); /* avoid division by zero (speed) */ filesize += !filesize; /* avoid division by zero (ratio) */ DISPLAYLEVEL(2, "\r%79s\r", ""); /* blank line */ DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n", filesize, compressedfilesize, (double)compressedfilesize / filesize * 100); { double const seconds = (double)(clockEnd - clockStart) / CLOCKS_PER_SEC; - DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds, + DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MiB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); } @@ -473,7 +486,7 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output free(in_buff); free(out_buff); fclose(finput); - if (strcmp(output_filename,stdoutmark)) fclose(foutput); /* do not close stdout */ + if (!LZ4IO_isStdout(output_filename)) fclose(foutput); /* do not close stdout */ return 0; } @@ -498,7 +511,7 @@ int LZ4IO_compressMultipleFilenames_Legacy( /* loop on each file */ for (i=0; i<ifntSize; i++) { size_t const ifnSize = strlen(inFileNamesTable[i]); - if (!strcmp(suffix, stdoutmark)) { + if (LZ4IO_isStdout(suffix)) { missed_files += LZ4IO_compressFilename_Legacy( inFileNamesTable[i], stdoutmark, compressionLevel, prefs); @@ -530,7 +543,6 @@ int LZ4IO_compressMultipleFilenames_Legacy( /********************************************* * Compression using Frame format *********************************************/ - typedef struct { void* srcBuffer; size_t srcBufferSize; @@ -551,15 +563,15 @@ static void* LZ4IO_createDict(size_t* dictSize, const char* const dictFilename) char* dictBuf; FILE* dictFile; - if (!circularBuf) EXM_THROW(25, "Allocation error : not enough memory for circular buffer"); - if (!dictFilename) EXM_THROW(25, "Dictionary error : no filename provided"); + if (!circularBuf) END_PROCESS(25, "Allocation error : not enough memory for circular buffer"); + if (!dictFilename) END_PROCESS(26, "Dictionary error : no filename provided"); dictFile = LZ4IO_openSrcFile(dictFilename); - if (!dictFile) EXM_THROW(25, "Dictionary error : could not open dictionary file"); + if (!dictFile) END_PROCESS(27, "Dictionary error : could not open dictionary file"); - /* opportunistically seek to the part of the file we care about. If this */ - /* fails it's not a problem since we'll just read everything anyways. */ - if (strcmp(dictFilename, stdinmark)) { + /* opportunistically seek to the part of the file we care about. + * If this fails it's not a problem since we'll just read everything anyways. */ + if (!LZ4IO_isStdin(dictFilename)) { (void)UTIL_fseek(dictFile, -LZ4_MAX_DICT_SIZE, SEEK_END); } @@ -584,7 +596,7 @@ static void* LZ4IO_createDict(size_t* dictSize, const char* const dictFilename) } else { /* Otherwise, we will alloc a new buffer and copy our dict into that. */ dictBuf = (char *)malloc(dictLen ? dictLen : 1); - if (!dictBuf) EXM_THROW(25, "Allocation error : not enough memory"); + if (!dictBuf) END_PROCESS(28, "Allocation error : not enough memory"); memcpy(dictBuf, circularBuf + dictStart, circularBufSize - dictStart); memcpy(dictBuf + circularBufSize - dictStart, circularBuf, dictLen - (circularBufSize - dictStart)); @@ -603,7 +615,7 @@ static LZ4F_CDict* LZ4IO_createCDict(const LZ4IO_prefs_t* const prefs) LZ4F_CDict* cdict; if (!prefs->useDictionary) return NULL; dictionaryBuffer = LZ4IO_createDict(&dictionarySize, prefs->dictionaryFilename); - if (!dictionaryBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary"); + if (!dictionaryBuffer) END_PROCESS(29, "Dictionary error : could not create dictionary"); cdict = LZ4F_createCDict(dictionaryBuffer, dictionarySize); free(dictionaryBuffer); return cdict; @@ -615,14 +627,14 @@ static cRess_t LZ4IO_createCResources(const LZ4IO_prefs_t* const prefs) cRess_t ress; LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&(ress.ctx), LZ4F_VERSION); - if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode)); + if (LZ4F_isError(errorCode)) END_PROCESS(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode)); /* Allocate Memory */ ress.srcBuffer = malloc(blockSize); ress.srcBufferSize = blockSize; ress.dstBufferSize = LZ4F_compressFrameBound(blockSize, NULL); /* cover worst case */ ress.dstBuffer = malloc(ress.dstBufferSize); - if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory"); + if (!ress.srcBuffer || !ress.dstBuffer) END_PROCESS(31, "Allocation error : not enough memory"); ress.cdict = LZ4IO_createCDict(prefs); @@ -638,7 +650,7 @@ static void LZ4IO_freeCResources(cRess_t ress) ress.cdict = NULL; { LZ4F_errorCode_t const errorCode = LZ4F_freeCompressionContext(ress.ctx); - if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); } + if (LZ4F_isError(errorCode)) END_PROCESS(35, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); } } /* @@ -686,7 +698,7 @@ LZ4IO_compressFilename_extRess(cRess_t ress, /* read first block */ readSize = fread(srcBuffer, (size_t)1, blockSize, srcFile); - if (ferror(srcFile)) EXM_THROW(30, "Error reading %s ", srcFileName); + if (ferror(srcFile)) END_PROCESS(40, "Error reading %s ", srcFileName); filesize += readSize; /* single-block file */ @@ -694,14 +706,14 @@ LZ4IO_compressFilename_extRess(cRess_t ress, /* Compress in single pass */ size_t const cSize = LZ4F_compressFrame_usingCDict(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs); if (LZ4F_isError(cSize)) - EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize)); + END_PROCESS(41, "Compression failed : %s", LZ4F_getErrorName(cSize)); compressedfilesize = cSize; - DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", + DISPLAYUPDATE(2, "\rRead : %u MiB ==> %.2f%% ", (unsigned)(filesize>>20), (double)compressedfilesize/(filesize+!filesize)*100); /* avoid division by zero */ /* Write Block */ if (fwrite(dstBuffer, 1, cSize, dstFile) != cSize) { - EXM_THROW(32, "Write error : failed writing single-block compressed frame"); + END_PROCESS(42, "Write error : failed writing single-block compressed frame"); } } else @@ -710,55 +722,55 @@ LZ4IO_compressFilename_extRess(cRess_t ress, { /* Write Frame Header */ size_t const headerSize = LZ4F_compressBegin_usingCDict(ctx, dstBuffer, dstBufferSize, ress.cdict, &prefs); - if (LZ4F_isError(headerSize)) EXM_THROW(33, "File header generation failed : %s", LZ4F_getErrorName(headerSize)); + if (LZ4F_isError(headerSize)) END_PROCESS(43, "File header generation failed : %s", LZ4F_getErrorName(headerSize)); if (fwrite(dstBuffer, 1, headerSize, dstFile) != headerSize) - EXM_THROW(34, "Write error : cannot write header"); + END_PROCESS(44, "Write error : cannot write header"); compressedfilesize += headerSize; /* Main Loop - one block at a time */ while (readSize>0) { size_t const outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL); if (LZ4F_isError(outSize)) - EXM_THROW(35, "Compression failed : %s", LZ4F_getErrorName(outSize)); + END_PROCESS(45, "Compression failed : %s", LZ4F_getErrorName(outSize)); compressedfilesize += outSize; - DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", + DISPLAYUPDATE(2, "\rRead : %u MiB ==> %.2f%% ", (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100); /* Write Block */ if (fwrite(dstBuffer, 1, outSize, dstFile) != outSize) - EXM_THROW(36, "Write error : cannot write compressed block"); + END_PROCESS(46, "Write error : cannot write compressed block"); /* Read next block */ readSize = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile); filesize += readSize; } - if (ferror(srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName); + if (ferror(srcFile)) END_PROCESS(47, "Error reading %s ", srcFileName); /* End of Frame mark */ { size_t const endSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL); if (LZ4F_isError(endSize)) - EXM_THROW(38, "End of frame error : %s", LZ4F_getErrorName(endSize)); + END_PROCESS(48, "End of frame error : %s", LZ4F_getErrorName(endSize)); if (fwrite(dstBuffer, 1, endSize, dstFile) != endSize) - EXM_THROW(39, "Write error : cannot write end of frame"); + END_PROCESS(49, "Write error : cannot write end of frame"); compressedfilesize += endSize; } } /* Release file handlers */ fclose (srcFile); - if (strcmp(dstFileName,stdoutmark)) fclose (dstFile); /* do not close stdout */ + if (!LZ4IO_isStdout(dstFileName)) fclose(dstFile); /* do not close stdout */ /* Copy owner, file permissions and modification time */ { stat_t statbuf; - if (strcmp (srcFileName, stdinmark) - && strcmp (dstFileName, stdoutmark) - && strcmp (dstFileName, nulmark) + if (!LZ4IO_isStdin(srcFileName) + && !LZ4IO_isStdout(dstFileName) + && !LZ4IO_isDevNull(dstFileName) && UTIL_getFileStat(srcFileName, &statbuf)) { UTIL_setFileStat(dstFileName, &statbuf); } } if (io_prefs->removeSrcFile) { /* remove source file : --rm */ if (remove(srcFileName)) - EXM_THROW(40, "Remove error : %s: %s", srcFileName, strerror(errno)); + END_PROCESS(50, "Remove error : %s: %s", srcFileName, strerror(errno)); } /* Final Status */ @@ -814,12 +826,13 @@ int LZ4IO_compressMultipleFilenames( /* loop on each file */ for (i=0; i<ifntSize; i++) { size_t const ifnSize = strlen(inFileNamesTable[i]); - if (!strcmp(suffix, stdoutmark)) { + if (LZ4IO_isStdout(suffix)) { missed_files += LZ4IO_compressFilename_extRess(ress, inFileNamesTable[i], stdoutmark, compressionLevel, prefs); continue; } + /* suffix != stdout => compress into a file => generate its name */ if (ofnSize <= ifnSize+suffixSize+1) { free(dstFileName); ofnSize = ifnSize + 20; @@ -877,14 +890,14 @@ LZ4IO_fwriteSparse(FILE* file, if (!sparseMode) { /* normal write */ size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); - if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : cannot write decoded block"); + if (sizeCheck != bufferSize) END_PROCESS(70, "Write error : cannot write decoded block"); return 0; } /* avoid int overflow */ if (storedSkips > 1 GB) { int const seekResult = UTIL_fseek(file, 1 GB, SEEK_CUR); - if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)"); + if (seekResult != 0) END_PROCESS(71, "1 GB skip error (sparse file support)"); storedSkips -= 1 GB; } @@ -901,13 +914,13 @@ LZ4IO_fwriteSparse(FILE* file, if (nb0T != seg0SizeT) { /* not all 0s */ errno = 0; { int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR); - if (seekResult) EXM_THROW(72, "Sparse skip error(%d): %s ; try --no-sparse", (int)errno, strerror(errno)); + if (seekResult) END_PROCESS(72, "Sparse skip error(%d): %s ; try --no-sparse", (int)errno, strerror(errno)); } storedSkips = 0; seg0SizeT -= nb0T; ptrT += nb0T; { size_t const sizeCheck = fwrite(ptrT, sizeT, seg0SizeT, file); - if (sizeCheck != seg0SizeT) EXM_THROW(73, "Write error : cannot write decoded block"); + if (sizeCheck != seg0SizeT) END_PROCESS(73, "Write error : cannot write decoded block"); } } ptrT += seg0SizeT; } @@ -921,10 +934,10 @@ LZ4IO_fwriteSparse(FILE* file, storedSkips += (unsigned) (restPtr - restStart); if (restPtr != restEnd) { int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR); - if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse"); + if (seekResult) END_PROCESS(74, "Sparse skip error ; try --no-sparse"); storedSkips = 0; { size_t const sizeCheck = fwrite(restPtr, 1, (size_t)(restEnd - restPtr), file); - if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(75, "Write error : cannot write decoded end of block"); + if (sizeCheck != (size_t)(restEnd - restPtr)) END_PROCESS(75, "Write error : cannot write decoded end of block"); } } } @@ -936,15 +949,17 @@ static void LZ4IO_fwriteSparseEnd(FILE* file, unsigned storedSkips) if (storedSkips>0) { /* implies sparseFileSupport>0 */ const char lastZeroByte[1] = { 0 }; if (UTIL_fseek(file, storedSkips-1, SEEK_CUR) != 0) - EXM_THROW(69, "Final skip error (sparse file)\n"); + END_PROCESS(68, "Final skip error (sparse file)\n"); if (fwrite(lastZeroByte, 1, 1, file) != 1) - EXM_THROW(69, "Write error : cannot write last zero\n"); + END_PROCESS(69, "Write error : cannot write last zero\n"); } } static unsigned g_magicRead = 0; /* out-parameter of LZ4IO_decodeLegacyStream() */ -static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput, const LZ4IO_prefs_t* prefs) + +static unsigned long long +LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput, const LZ4IO_prefs_t* prefs) { unsigned long long streamSize = 0; unsigned storedSkips = 0; @@ -952,18 +967,19 @@ static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput, /* Allocate Memory */ char* const in_buff = (char*)malloc((size_t)LZ4_compressBound(LEGACY_BLOCKSIZE)); char* const out_buff = (char*)malloc(LEGACY_BLOCKSIZE); - if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory"); + if (!in_buff || !out_buff) END_PROCESS(51, "Allocation error : not enough memory"); /* Main Loop */ while (1) { unsigned int blockSize; /* Block Size */ - { size_t const sizeCheck = fread(in_buff, 1, 4, finput); + { size_t const sizeCheck = fread(in_buff, 1, LZ4IO_LEGACY_BLOCK_HEADER_SIZE, finput); if (sizeCheck == 0) break; /* Nothing to read : file read is completed */ - if (sizeCheck != 4) EXM_THROW(52, "Read error : cannot access block size "); } - blockSize = LZ4IO_readLE32(in_buff); /* Convert to Little Endian */ - if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) { + if (sizeCheck != LZ4IO_LEGACY_BLOCK_HEADER_SIZE) END_PROCESS(52, "Read error : cannot access block size "); + } + blockSize = LZ4IO_readLE32(in_buff); /* Convert to Little Endian */ + if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) { /* Cannot read next block : maybe new stream ? */ g_magicRead = blockSize; break; @@ -971,16 +987,16 @@ static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput, /* Read Block */ { size_t const sizeCheck = fread(in_buff, 1, blockSize, finput); - if (sizeCheck!=blockSize) EXM_THROW(52, "Read error : cannot access compressed block !"); } + if (sizeCheck != blockSize) END_PROCESS(53, "Read error : cannot access compressed block !"); } /* Decode Block */ { int const decodeSize = LZ4_decompress_safe(in_buff, out_buff, (int)blockSize, LEGACY_BLOCKSIZE); - if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !"); + if (decodeSize < 0) END_PROCESS(54, "Decoding Failed ! Corrupted input detected !"); streamSize += (unsigned long long)decodeSize; /* Write Block */ storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, (size_t)decodeSize, prefs->sparseFileSupport, storedSkips); /* success or die */ } } - if (ferror(finput)) EXM_THROW(54, "Read error : ferror"); + if (ferror(finput)) END_PROCESS(55, "Read error : ferror"); LZ4IO_fwriteSparseEnd(foutput, storedSkips); @@ -1013,7 +1029,7 @@ static void LZ4IO_loadDDict(dRess_t* ress, const LZ4IO_prefs_t* const prefs) } ress->dictBuffer = LZ4IO_createDict(&ress->dictBufferSize, prefs->dictionaryFilename); - if (!ress->dictBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary"); + if (!ress->dictBuffer) END_PROCESS(25, "Dictionary error : could not create dictionary"); } static const size_t LZ4IO_dBufferSize = 64 KB; @@ -1023,14 +1039,14 @@ static dRess_t LZ4IO_createDResources(const LZ4IO_prefs_t* const prefs) /* init */ LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&ress.dCtx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) EXM_THROW(60, "Can't create LZ4F context : %s", LZ4F_getErrorName(errorCode)); + if (LZ4F_isError(errorCode)) END_PROCESS(60, "Can't create LZ4F context : %s", LZ4F_getErrorName(errorCode)); /* Allocate Memory */ ress.srcBufferSize = LZ4IO_dBufferSize; ress.srcBuffer = malloc(ress.srcBufferSize); ress.dstBufferSize = LZ4IO_dBufferSize; ress.dstBuffer = malloc(ress.dstBufferSize); - if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory"); + if (!ress.srcBuffer || !ress.dstBuffer) END_PROCESS(61, "Allocation error : not enough memory"); LZ4IO_loadDDict(&ress, prefs); @@ -1041,7 +1057,7 @@ static dRess_t LZ4IO_createDResources(const LZ4IO_prefs_t* const prefs) static void LZ4IO_freeDResources(dRess_t ress) { LZ4F_errorCode_t errorCode = LZ4F_freeDecompressionContext(ress.dCtx); - if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); + if (LZ4F_isError(errorCode)) END_PROCESS(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); free(ress.srcBuffer); free(ress.dstBuffer); free(ress.dictBuffer); @@ -1056,13 +1072,22 @@ LZ4IO_decompressLZ4F(dRess_t ress, unsigned long long filesize = 0; LZ4F_errorCode_t nextToLoad; unsigned storedSkips = 0; + LZ4F_decompressOptions_t const dOpt_skipCrc = { 0, 1, 0, 0 }; + const LZ4F_decompressOptions_t* const dOptPtr = + ((prefs->blockChecksum==0) && (prefs->streamChecksum==0)) ? + &dOpt_skipCrc : NULL; /* Init feed with magic number (already consumed from FILE* sFile) */ { size_t inSize = MAGICNUMBER_SIZE; size_t outSize= 0; LZ4IO_writeLE32(ress.srcBuffer, LZ4IO_MAGICNUMBER); - nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, ress.dstBuffer, &outSize, ress.srcBuffer, &inSize, ress.dictBuffer, ress.dictBufferSize, NULL); - if (LZ4F_isError(nextToLoad)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(nextToLoad)); + nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, + ress.dstBuffer, &outSize, + ress.srcBuffer, &inSize, + ress.dictBuffer, ress.dictBufferSize, + dOptPtr); /* set it once, it's enough */ + if (LZ4F_isError(nextToLoad)) + END_PROCESS(62, "Header error : %s", LZ4F_getErrorName(nextToLoad)); } /* Main Loop */ @@ -1080,8 +1105,13 @@ LZ4IO_decompressLZ4F(dRess_t ress, /* Decode Input (at least partially) */ size_t remaining = readSize - pos; decodedBytes = ress.dstBufferSize; - nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, ress.dictBuffer, ress.dictBufferSize, NULL); - if (LZ4F_isError(nextToLoad)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad)); + nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, + ress.dstBuffer, &decodedBytes, + (char*)(ress.srcBuffer)+pos, &remaining, + ress.dictBuffer, ress.dictBufferSize, + NULL); + if (LZ4F_isError(nextToLoad)) + END_PROCESS(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad)); pos += remaining; /* Write Block */ @@ -1089,17 +1119,17 @@ LZ4IO_decompressLZ4F(dRess_t ress, if (!prefs->testMode) storedSkips = LZ4IO_fwriteSparse(dstFile, ress.dstBuffer, decodedBytes, prefs->sparseFileSupport, storedSkips); filesize += decodedBytes; - DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20)); + DISPLAYUPDATE(2, "\rDecompressed : %u MiB ", (unsigned)(filesize>>20)); } if (!nextToLoad) break; } } /* can be out because readSize == 0, which could be an fread() error */ - if (ferror(srcFile)) EXM_THROW(67, "Read error"); + if (ferror(srcFile)) END_PROCESS(67, "Read error"); if (!prefs->testMode) LZ4IO_fwriteSparseEnd(dstFile, storedSkips); - if (nextToLoad!=0) EXM_THROW(68, "Unfinished stream"); + if (nextToLoad!=0) END_PROCESS(68, "Unfinished stream"); return filesize; } @@ -1123,19 +1153,36 @@ LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigned storedSkips = 0; if (fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput) != MAGICNUMBER_SIZE) { - EXM_THROW(50, "Pass-through write error"); + END_PROCESS(50, "Pass-through write error"); } while (readBytes) { readBytes = fread(buffer, 1, sizeof(buffer), finput); total += readBytes; storedSkips = LZ4IO_fwriteSparse(foutput, buffer, readBytes, sparseFileSupport, storedSkips); } - if (ferror(finput)) EXM_THROW(51, "Read Error"); + if (ferror(finput)) END_PROCESS(51, "Read Error"); LZ4IO_fwriteSparseEnd(foutput, storedSkips); return total; } +/* when fseek() doesn't work (pipe scenario), + * read and forget from input. +**/ +#define SKIP_BUFF_SIZE (16 KB) +#define MIN(a,b) ( ((a)<(b)) ? (a) : (b) ) +static int skipStream(FILE* f, unsigned offset) +{ + char buf[SKIP_BUFF_SIZE]; + while (offset > 0) { + size_t const tr = MIN(offset, sizeof(buf)); + size_t const r = fread(buf, 1, tr, f); + if (r != tr) return 1; /* error reading f */ + offset -= (unsigned)tr; + } + assert(offset == 0); + return 0; +} /** Safely handle cases when (unsigned)offset > LONG_MAX */ static int fseek_u32(FILE *fp, unsigned offset, int where) @@ -1147,14 +1194,17 @@ static int fseek_u32(FILE *fp, unsigned offset, int where) while (offset > 0) { unsigned s = offset; if (s > stepMax) s = stepMax; - errorNb = UTIL_fseek(fp, (long) s, SEEK_CUR); - if (errorNb != 0) break; - offset -= s; + errorNb = UTIL_fseek(fp, (long)s, SEEK_CUR); + if (errorNb==0) { offset -= s; continue; } + errorNb = skipStream(fp, offset); + offset = 0; } return errorNb; } + #define ENDOFSTREAM ((unsigned long long)-1) +#define DECODING_ERROR ((unsigned long long)-2) static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutput, @@ -1175,7 +1225,7 @@ selectDecoder(dRess_t ress, size_t const nbReadBytes = fread(MNstore, 1, MAGICNUMBER_SIZE, finput); if (nbReadBytes==0) { nbFrames = 0; return ENDOFSTREAM; } /* EOF */ if (nbReadBytes != MAGICNUMBER_SIZE) - EXM_THROW(40, "Unrecognized header : Magic Number unreadable"); + END_PROCESS(40, "Unrecognized header : Magic Number unreadable"); magicNumber = LZ4IO_readLE32(MNstore); /* Little Endian format */ } if (LZ4IO_isSkippableMagicNumber(magicNumber)) @@ -1192,15 +1242,14 @@ selectDecoder(dRess_t ress, DISPLAYLEVEL(4, "Skipping detected skippable area \n"); { size_t const nbReadBytes = fread(MNstore, 1, 4, finput); if (nbReadBytes != 4) - EXM_THROW(42, "Stream error : skippable size unreadable"); + END_PROCESS(42, "Stream error : skippable size unreadable"); } { unsigned const size = LZ4IO_readLE32(MNstore); int const errorNb = fseek_u32(finput, size, SEEK_CUR); if (errorNb != 0) - EXM_THROW(43, "Stream error : cannot skip skippable area"); + END_PROCESS(43, "Stream error : cannot skip skippable area"); } return 0; - EXTENDED_FORMAT; /* macro extension for custom formats */ default: if (nbFrames == 1) { /* just started */ /* Wrong magic number at the beginning of 1st stream */ @@ -1208,7 +1257,7 @@ selectDecoder(dRess_t ress, nbFrames = 0; return LZ4IO_passThrough(finput, foutput, MNstore, prefs->sparseFileSupport); } - EXM_THROW(44,"Unrecognized header : file cannot be decoded"); + END_PROCESS(44,"Unrecognized header : file cannot be decoded"); } { long int const position = ftell(finput); /* only works for files < 2 GB */ DISPLAYLEVEL(2, "Stream followed by undecodable data "); @@ -1216,7 +1265,7 @@ selectDecoder(dRess_t ress, DISPLAYLEVEL(2, "at position %i ", (int)position); DISPLAYLEVEL(2, "\n"); } - return ENDOFSTREAM; + return DECODING_ERROR; } } @@ -1228,6 +1277,7 @@ LZ4IO_decompressSrcFile(dRess_t ress, { FILE* const foutput = ress.dstFile; unsigned long long filesize = 0; + int result = 0; /* Init */ FILE* const finput = LZ4IO_openSrcFile(input_filename); @@ -1239,6 +1289,7 @@ LZ4IO_decompressSrcFile(dRess_t ress, unsigned long long const decodedSize = selectDecoder(ress, finput, foutput, prefs); if (decodedSize == ENDOFSTREAM) break; + if (decodedSize == DECODING_ERROR) { result=1; break; } filesize += decodedSize; } @@ -1246,7 +1297,7 @@ LZ4IO_decompressSrcFile(dRess_t ress, fclose(finput); if (prefs->removeSrcFile) { /* --rm */ if (remove(input_filename)) - EXM_THROW(45, "Remove error : %s: %s", input_filename, strerror(errno)); + END_PROCESS(45, "Remove error : %s: %s", input_filename, strerror(errno)); } /* Final Status */ @@ -1254,7 +1305,7 @@ LZ4IO_decompressSrcFile(dRess_t ress, DISPLAYLEVEL(2, "%-20.20s : decoded %llu bytes \n", input_filename, filesize); (void)output_filename; - return 0; + return result; } @@ -1263,45 +1314,50 @@ LZ4IO_decompressDstFile(dRess_t ress, const char* input_filename, const char* output_filename, const LZ4IO_prefs_t* const prefs) { + int result; stat_t statbuf; int stat_result = 0; FILE* const foutput = LZ4IO_openDstFile(output_filename, prefs); if (foutput==NULL) return 1; /* failure */ - if ( strcmp(input_filename, stdinmark) + if ( !LZ4IO_isStdin(input_filename) && UTIL_getFileStat(input_filename, &statbuf)) stat_result = 1; ress.dstFile = foutput; - LZ4IO_decompressSrcFile(ress, input_filename, output_filename, prefs); + result = LZ4IO_decompressSrcFile(ress, input_filename, output_filename, prefs); fclose(foutput); /* Copy owner, file permissions and modification time */ if ( stat_result != 0 - && strcmp (output_filename, stdoutmark) - && strcmp (output_filename, nulmark)) { + && !LZ4IO_isStdout(output_filename) + && !LZ4IO_isDevNull(output_filename)) { UTIL_setFileStat(output_filename, &statbuf); /* should return value be read ? or is silent fail good enough ? */ } - return 0; + return result; } +/* Note : LZ4IO_decompressFilename() + * can provide total decompression time for the specified fileName. + * This information is not available with LZ4IO_decompressMultipleFilenames(). + */ int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename, const LZ4IO_prefs_t* prefs) { dRess_t const ress = LZ4IO_createDResources(prefs); clock_t const start = clock(); - int const missingFiles = LZ4IO_decompressDstFile(ress, input_filename, output_filename, prefs); + int const status = LZ4IO_decompressDstFile(ress, input_filename, output_filename, prefs); clock_t const end = clock(); double const seconds = (double)(end - start) / CLOCKS_PER_SEC; DISPLAYLEVEL(4, "Done in %.2f sec \n", seconds); LZ4IO_freeDResources(ress); - return missingFiles; + return status; } @@ -1318,23 +1374,26 @@ int LZ4IO_decompressMultipleFilenames( size_t const suffixSize = strlen(suffix); dRess_t ress = LZ4IO_createDResources(prefs); - if (outFileName==NULL) EXM_THROW(70, "Memory allocation error"); + if (outFileName==NULL) END_PROCESS(70, "Memory allocation error"); + if (prefs->blockChecksum==0 && prefs->streamChecksum==0) { + DISPLAYLEVEL(4, "disabling checksum validation during decoding \n"); + } ress.dstFile = LZ4IO_openDstFile(stdoutmark, prefs); for (i=0; i<ifntSize; i++) { size_t const ifnSize = strlen(inFileNamesTable[i]); const char* const suffixPtr = inFileNamesTable[i] + ifnSize - suffixSize; - if (!strcmp(suffix, stdoutmark)) { - missingFiles += LZ4IO_decompressSrcFile(ress, inFileNamesTable[i], stdoutmark, prefs); + if (LZ4IO_isStdout(suffix) || LZ4IO_isDevNull(suffix)) { + missingFiles += LZ4IO_decompressSrcFile(ress, inFileNamesTable[i], suffix, prefs); continue; } if (ofnSize <= ifnSize-suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); - if (outFileName==NULL) EXM_THROW(71, "Memory allocation error"); + if (outFileName==NULL) END_PROCESS(71, "Memory allocation error"); } - if (ifnSize <= suffixSize || strcmp(suffixPtr, suffix) != 0) { + if (ifnSize <= suffixSize || !UTIL_sameString(suffixPtr, suffix) ) { DISPLAYLEVEL(1, "File extension doesn't match expected LZ4_EXTENSION (%4s); will not process file: %s\n", suffix, inFileNamesTable[i]); skippedFiles++; continue; @@ -1387,7 +1446,7 @@ static const char * LZ4IO_frameTypeNames[] = {"LZ4Frame", "LegacyFrame", "Skippa /* Read block headers and skip block data Return total blocks size for this frame including block headers, block checksums and content checksums. - returns 0 in case it can't succesfully skip block data. + returns 0 in case it can't successfully skip block data. Assumes SEEK_CUR after frame header. */ static unsigned long long @@ -1424,37 +1483,47 @@ LZ4IO_skipBlocksData(FILE* finput, return totalBlocksSize; } +static const unsigned long long legacyFrameUndecodable = (0ULL-1); /* For legacy frames only. Read block headers and skip block data. Return total blocks size for this frame including block headers. - or 0 in case it can't succesfully skip block data. + or legacyFrameUndecodable in case it can't successfully skip block data. This works as long as legacy block header size = magic number size. Assumes SEEK_CUR after frame header. */ static unsigned long long LZ4IO_skipLegacyBlocksData(FILE* finput) { - unsigned char blockInfo[LZIO_LEGACY_BLOCK_HEADER_SIZE]; + unsigned char blockInfo[LZ4IO_LEGACY_BLOCK_HEADER_SIZE]; unsigned long long totalBlocksSize = 0; - LZ4IO_STATIC_ASSERT(LZIO_LEGACY_BLOCK_HEADER_SIZE == MAGICNUMBER_SIZE); + LZ4IO_STATIC_ASSERT(LZ4IO_LEGACY_BLOCK_HEADER_SIZE == MAGICNUMBER_SIZE); for (;;) { - if (!fread(blockInfo, 1, LZIO_LEGACY_BLOCK_HEADER_SIZE, finput)) { + size_t const bhs = fread(blockInfo, 1, LZ4IO_LEGACY_BLOCK_HEADER_SIZE, finput); + if (bhs == 0) { if (feof(finput)) return totalBlocksSize; - return 0; + return legacyFrameUndecodable; + } + if (bhs != 4) { + return legacyFrameUndecodable; } { const unsigned int nextCBlockSize = LZ4IO_readLE32(&blockInfo); - if ( nextCBlockSize == LEGACY_MAGICNUMBER || - nextCBlockSize == LZ4IO_MAGICNUMBER || - LZ4IO_isSkippableMagicNumber(nextCBlockSize)) { - /* Rewind back. we want cursor at the begining of next frame.*/ - if (fseek(finput, -LZIO_LEGACY_BLOCK_HEADER_SIZE, SEEK_CUR) != 0) { - return 0; + if ( nextCBlockSize == LEGACY_MAGICNUMBER + || nextCBlockSize == LZ4IO_MAGICNUMBER + || LZ4IO_isSkippableMagicNumber(nextCBlockSize) ) { + /* Rewind back. we want cursor at the beginning of next frame */ + if (UTIL_fseek(finput, -LZ4IO_LEGACY_BLOCK_HEADER_SIZE, SEEK_CUR) != 0) { + END_PROCESS(37, "impossible to skip backward"); } break; } - totalBlocksSize += LZIO_LEGACY_BLOCK_HEADER_SIZE + nextCBlockSize; - /* skip to the next block */ + if (nextCBlockSize > LZ4IO_LEGACY_BLOCK_SIZE_MAX) { + DISPLAYLEVEL(4, "Error : block in legacy frame is too large \n"); + return legacyFrameUndecodable; + } + totalBlocksSize += LZ4IO_LEGACY_BLOCK_HEADER_SIZE + nextCBlockSize; + /* skip to the next block + * note : this won't fail if nextCBlockSize is too large, skipping past the end of finput */ if (UTIL_fseek(finput, nextCBlockSize, SEEK_CUR) != 0) { - return 0; + return legacyFrameUndecodable; } } } return totalBlocksSize; } @@ -1514,7 +1583,7 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam if (nbReadBytes == 0) { break; } /* EOF */ result = LZ4IO_format_not_known; /* default result (error) */ if (nbReadBytes != MAGICNUMBER_SIZE) { - EXM_THROW(40, "Unrecognized header : Magic Number unreadable"); + END_PROCESS(40, "Unrecognized header : Magic Number unreadable"); } } magicNumber = LZ4IO_readLE32(buffer); /* Little Endian format */ if (LZ4IO_isSkippableMagicNumber(magicNumber)) @@ -1525,14 +1594,14 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam if (cfinfo->frameSummary.frameType != lz4Frame) cfinfo->eqFrameTypes = 0; /* Get frame info */ { const size_t readBytes = fread(buffer + MAGICNUMBER_SIZE, 1, LZ4F_HEADER_SIZE_MIN - MAGICNUMBER_SIZE, finput); - if (!readBytes || ferror(finput)) EXM_THROW(71, "Error reading %s", input_filename); + if (!readBytes || ferror(finput)) END_PROCESS(71, "Error reading %s", input_filename); } { size_t hSize = LZ4F_headerSize(&buffer, LZ4F_HEADER_SIZE_MIN); if (LZ4F_isError(hSize)) break; if (hSize > (LZ4F_HEADER_SIZE_MIN + MAGICNUMBER_SIZE)) { /* We've already read LZ4F_HEADER_SIZE_MIN so read any extra until hSize*/ const size_t readBytes = fread(buffer + LZ4F_HEADER_SIZE_MIN, 1, hSize - LZ4F_HEADER_SIZE_MIN, finput); - if (!readBytes || ferror(finput)) EXM_THROW(72, "Error reading %s", input_filename); + if (!readBytes || ferror(finput)) END_PROCESS(72, "Error reading %s", input_filename); } /* Create decompression context */ { LZ4F_dctx* dctx; @@ -1578,6 +1647,11 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam cfinfo->eqBlockTypes = 0; cfinfo->allContentSize = 0; { const unsigned long long totalBlocksSize = LZ4IO_skipLegacyBlocksData(finput); + if (totalBlocksSize == legacyFrameUndecodable) { + DISPLAYLEVEL(1, "Corrupted legacy frame \n"); + result = LZ4IO_format_not_known; + break; + } if (totalBlocksSize) { DISPLAYLEVEL(3, " %6llu %14s %5s %8s %20llu %20s %9s\n", cfinfo->frameCount + 1, @@ -1595,12 +1669,12 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam cfinfo->allContentSize = 0; { size_t const nbReadBytes = fread(buffer, 1, 4, finput); if (nbReadBytes != 4) - EXM_THROW(42, "Stream error : skippable size unreadable"); + END_PROCESS(42, "Stream error : skippable size unreadable"); } { unsigned const size = LZ4IO_readLE32(buffer); int const errorNb = fseek_u32(finput, size, SEEK_CUR); if (errorNb != 0) - EXM_THROW(43, "Stream error : cannot skip skippable area"); + END_PROCESS(43, "Stream error : cannot skip skippable area"); DISPLAYLEVEL(3, " %6llu %14s %5s %8s %20u %20s %9s\n", cfinfo->frameCount + 1, "SkippableFrame", @@ -1614,6 +1688,7 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam DISPLAYLEVEL(3, "Stream followed by undecodable data "); if (position != -1L) DISPLAYLEVEL(3, "at position %i ", (int)position); + result = LZ4IO_format_not_known; DISPLAYLEVEL(3, "\n"); } break; @@ -1639,9 +1714,9 @@ int LZ4IO_displayCompressedFilesInfo(const char** inFileNames, size_t ifnIdx) /* Get file info */ LZ4IO_cFileInfo_t cfinfo = LZ4IO_INIT_CFILEINFO; cfinfo.fileName = LZ4IO_baseName(inFileNames[idx]); - if (!UTIL_isRegFile(inFileNames[idx])) { + if (LZ4IO_isStdin(inFileNames[idx]) ? !UTIL_isRegFD(0) : !UTIL_isRegFile(inFileNames[idx])) { DISPLAYLEVEL(1, "lz4: %s is not a regular file \n", inFileNames[idx]); - return 0; + return 1; } DISPLAYLEVEL(3, "%s(%llu/%llu)\n", cfinfo.fileName, (unsigned long long)idx + 1, (unsigned long long)ifnIdx); DISPLAYLEVEL(3, " %6s %14s %5s %8s %20s %20s %9s\n", @@ -1650,7 +1725,7 @@ int LZ4IO_displayCompressedFilesInfo(const char** inFileNames, size_t ifnIdx) if (op_result != LZ4IO_LZ4F_OK) { assert(op_result == LZ4IO_format_not_known); DISPLAYLEVEL(1, "lz4: %s: File format not recognized \n", inFileNames[idx]); - return 0; + return 1; } } DISPLAYLEVEL(3, "\n"); if (g_displayLevel < 3) { diff --git a/programs/lz4io.h b/programs/lz4io.h index d6d7eee..0cfb1d2 100644 --- a/programs/lz4io.h +++ b/programs/lz4io.h @@ -1,6 +1,6 @@ /* LZ4io.h - LZ4 File/Stream Interface - Copyright (C) Yann Collet 2011-2016 + Copyright (C) Yann Collet 2011-2020 GPL v2 License This program is free software; you can redistribute it and/or modify @@ -57,8 +57,6 @@ typedef struct LZ4IO_prefs_s LZ4IO_prefs_t; LZ4IO_prefs_t* LZ4IO_defaultPreferences(void); void LZ4IO_freePreferences(LZ4IO_prefs_t* prefs); -/* Size in bytes of a legacy block header in little-endian format */ -#define LZIO_LEGACY_BLOCK_HEADER_SIZE 4 /* ************************************************** */ /* ****************** Functions ********************* */ diff --git a/programs/platform.h b/programs/platform.h index ab8300d..43a171b 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -1,6 +1,6 @@ /* platform.h - compiler and OS detection - Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet + Copyright (C) 2016-2020, Przemyslaw Skibinski, Yann Collet This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -80,7 +80,7 @@ extern "C" { ************************************************************** */ #if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \ || defined(__midipix__) || defined(__VMS)) -# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1–2001 (SUSv3) conformant */ \ +# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1-2001 (SUSv3) conformant */ \ || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__MidnightBSD__) /* BSD distros */ \ || defined(__HAIKU__) # define PLATFORM_POSIX_VERSION 200112L diff --git a/programs/util.h b/programs/util.h index 733c1ca..3192ddc 100644 --- a/programs/util.h +++ b/programs/util.h @@ -1,6 +1,6 @@ /* util.h - utility functions - Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet + Copyright (C) 2016-2020, Przemyslaw Skibinski, Yann Collet This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -175,6 +175,39 @@ extern "C" { #endif + +/*-**************************************** +* Allocation functions +******************************************/ +/* + * A modified version of realloc(). + * If UTIL_realloc() fails the original block is freed. +*/ +UTIL_STATIC void* UTIL_realloc(void* ptr, size_t size) +{ + void* const newptr = realloc(ptr, size); + if (newptr) return newptr; + free(ptr); + return NULL; +} + + +/*-**************************************** +* String functions +******************************************/ +/* + * A modified version of realloc(). + * If UTIL_realloc() fails the original block is freed. +*/ +UTIL_STATIC int UTIL_sameString(const char* a, const char* b) +{ + assert(a!=NULL && b!=NULL); /* unsupported scenario */ + if (a==NULL) return 0; + if (b==NULL) return 0; + return !strcmp(a,b); +} + + /*-**************************************** * Time functions ******************************************/ @@ -317,6 +350,7 @@ UTIL_STATIC void UTIL_waitForNextTick(void) UTIL_STATIC int UTIL_isRegFile(const char* infilename); +UTIL_STATIC int UTIL_isRegFD(int fd); UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf) @@ -333,7 +367,8 @@ UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf) timebuf.modtime = statbuf->st_mtime; res += utime(filename, &timebuf); /* set access and modification times */ #else - struct timespec timebuf[2] = {}; + struct timespec timebuf[2]; + memset(timebuf, 0, sizeof(timebuf)); timebuf[0].tv_nsec = UTIME_NOW; timebuf[1].tv_sec = statbuf->st_mtime; res += utimensat(AT_FDCWD, filename, timebuf, 0); /* set access and modification times */ @@ -351,6 +386,19 @@ UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf) } +UTIL_STATIC int UTIL_getFDStat(int fd, stat_t *statbuf) +{ + int r; +#if defined(_MSC_VER) + r = _fstat64(fd, statbuf); + if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */ +#else + r = fstat(fd, statbuf); + if (r || !S_ISREG(statbuf->st_mode)) return 0; /* No good... */ +#endif + return 1; +} + UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf) { int r; @@ -365,6 +413,17 @@ UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf) } +UTIL_STATIC int UTIL_isRegFD(int fd) +{ + stat_t statbuf; +#ifdef _WIN32 + /* Windows runtime library always open file descriptors 0, 1 and 2 in text mode, therefore we can't use them for binary I/O */ + if(fd < 3) return 0; +#endif + return UTIL_getFDStat(fd, &statbuf); /* Only need to know whether it is a regular file */ +} + + UTIL_STATIC int UTIL_isRegFile(const char* infilename) { stat_t statbuf; @@ -425,19 +484,6 @@ UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFi } -/* - * A modified version of realloc(). - * If UTIL_realloc() fails the original block is freed. -*/ -UTIL_STATIC void* UTIL_realloc(void* ptr, size_t size) -{ - void* const newptr = realloc(ptr, size); - if (newptr) return newptr; - free(ptr); - return NULL; -} - - #ifdef _WIN32 # define UTIL_HAS_CREATEFILELIST @@ -511,22 +557,23 @@ UTIL_STATIC int UTIL_prepareFileList(const char* dirName, char** bufStart, size_ { DIR* dir; struct dirent * entry; - int dirLength, nbFiles = 0; + size_t dirLength; + int nbFiles = 0; if (!(dir = opendir(dirName))) { fprintf(stderr, "Cannot open directory '%s': %s\n", dirName, strerror(errno)); return 0; } - dirLength = (int)strlen(dirName); + dirLength = strlen(dirName); errno = 0; while ((entry = readdir(dir)) != NULL) { char* path; - int fnameLength, pathLength; + size_t fnameLength, pathLength; if (strcmp (entry->d_name, "..") == 0 || strcmp (entry->d_name, ".") == 0) continue; - fnameLength = (int)strlen(entry->d_name); - path = (char*) malloc(dirLength + fnameLength + 2); + fnameLength = strlen(entry->d_name); + path = (char*)malloc(dirLength + fnameLength + 2); if (!path) { closedir(dir); return 0; } memcpy(path, dirName, dirLength); path[dirLength] = '/'; @@ -539,7 +586,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char* dirName, char** bufStart, size_ if (*bufStart == NULL) { free(path); closedir(dir); return 0; } } else { if (*bufStart + *pos + pathLength >= *bufEnd) { - ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; + size_t const newListSize = (size_t)(*bufEnd - *bufStart) + LIST_SIZE_INCREASE; *bufStart = (char*)UTIL_realloc(*bufStart, newListSize); *bufEnd = *bufStart + newListSize; if (*bufStart == NULL) { free(path); closedir(dir); return 0; } @@ -638,8 +685,8 @@ UTIL_createFileList(const char** inputNames, unsigned inputNamesNb, UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer) { - if (allocatedBuffer) free(allocatedBuffer); - if (filenameTable) free((void*)filenameTable); + free(allocatedBuffer); + free((void*)filenameTable); } |