summaryrefslogtreecommitdiffstats
path: root/programs
diff options
context:
space:
mode:
Diffstat (limited to 'programs')
-rw-r--r--programs/Makefile69
-rw-r--r--programs/bench.c441
-rw-r--r--programs/bench.h29
-rw-r--r--programs/datagen.c2
-rw-r--r--programs/datagen.h2
-rw-r--r--programs/lz4-exe.rc.in3
-rw-r--r--programs/lz4.116
-rw-r--r--programs/lz4.1.md14
-rw-r--r--programs/lz4cli.c104
-rw-r--r--programs/lz4io.c389
-rw-r--r--programs/lz4io.h4
-rw-r--r--programs/platform.h4
-rw-r--r--programs/util.h93
13 files changed, 708 insertions, 462 deletions
diff --git a/programs/Makefile b/programs/Makefile
index c1053f6..ace0d03 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -1,6 +1,6 @@
# ##########################################################################
# LZ4 programs - Makefile
-# Copyright (C) Yann Collet 2011-2017
+# Copyright (C) Yann Collet 2011-2020
#
# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
#
@@ -28,13 +28,14 @@
# lz4c : CLU, supporting also legacy lz4demo arguments
# lz4c32: Same as lz4c, but forced to compile in 32-bits mode
# ##########################################################################
+SED = sed
# Version numbers
LZ4DIR := ../lib
LIBVER_SRC := $(LZ4DIR)/lz4.h
-LIBVER_MAJOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
-LIBVER_MINOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
-LIBVER_PATCH_SCRIPT:=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
+LIBVER_MAJOR_SCRIPT:=`$(SED) -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
+LIBVER_MINOR_SCRIPT:=`$(SED) -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
+LIBVER_PATCH_SCRIPT:=`$(SED) -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT))
LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
@@ -51,16 +52,26 @@ DEBUGFLAGS= -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
-Wpointer-arith -Wstrict-aliasing=1
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
+
+include ../Makefile.inc
+
+OS_VERSION ?= $(UNAME) -r
+ifeq ($(TARGET_OS)$(shell $(OS_VERSION)),SunOS5.10)
+LDFLAGS += -lrt
+endif
+
FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
LZ4_VERSION=$(LIBVER)
MD2ROFF = ronn
MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="lz4 $(LZ4_VERSION)"
-include ../Makefile.inc
default: lz4-release
+# silent mode by default; verbose can be triggered by V=1 or VERBOSE=1
+$(V)$(VERBOSE).SILENT:
+
all: lz4 lz4c
all32: CFLAGS+=-m32
@@ -69,7 +80,7 @@ all32: all
ifeq ($(WINBASED),yes)
lz4-exe.rc: lz4-exe.rc.in
@echo creating executable resource
- $(Q)sed -e 's|@PROGNAME@|lz4|' \
+ $(SED) -e 's|@PROGNAME@|lz4|' \
-e 's|@LIBVER_MAJOR@|$(LIBVER_MAJOR)|g' \
-e 's|@LIBVER_MINOR@|$(LIBVER_MINOR)|g' \
-e 's|@LIBVER_PATCH@|$(LIBVER_PATCH)|g' \
@@ -110,7 +121,7 @@ lz4c32 : $(SRCFILES)
$(CC) $(FLAGS) $^ -o $@$(EXT)
lz4.1: lz4.1.md $(LIBVER_SRC)
- cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
+ cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | $(SED) -n '/^\.\\\".*/!p' > $@
man: lz4.1
@@ -122,10 +133,10 @@ preview-man: clean-man man
clean:
ifeq ($(WINBASED),yes)
- $(Q)$(RM) *.rc
+ $(RM) *.rc
endif
- @$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
- @$(RM) core *.o *.test tmp* \
+ $(MAKE) -C $(LZ4DIR) $@ > $(VOID)
+ $(RM) core *.o *.test tmp* \
lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) lz4-wlib$(EXT) \
unlz4$(EXT) lz4cat$(EXT)
@echo Cleaning completed
@@ -160,28 +171,28 @@ MAN1DIR ?= $(mandir)/man1
man1dir ?= $(MAN1DIR)
install: lz4
- @echo Installing binaries
- @$(INSTALL_DIR) $(DESTDIR)$(bindir)/ $(DESTDIR)$(man1dir)/
- @$(INSTALL_PROGRAM) lz4$(EXT) $(DESTDIR)$(bindir)/lz4$(EXT)
- @$(LN_S) lz4$(EXT) $(DESTDIR)$(bindir)/lz4c$(EXT)
- @$(LN_S) lz4$(EXT) $(DESTDIR)$(bindir)/lz4cat$(EXT)
- @$(LN_S) lz4$(EXT) $(DESTDIR)$(bindir)/unlz4$(EXT)
- @echo Installing man pages
- @$(INSTALL_DATA) lz4.1 $(DESTDIR)$(man1dir)/lz4.1
- @$(LN_SF) lz4.1 $(DESTDIR)$(man1dir)/lz4c.1
- @$(LN_SF) lz4.1 $(DESTDIR)$(man1dir)/lz4cat.1
- @$(LN_SF) lz4.1 $(DESTDIR)$(man1dir)/unlz4.1
+ @echo Installing binaries in $(DESTDIR)$(bindir)
+ $(INSTALL_DIR) $(DESTDIR)$(bindir)/ $(DESTDIR)$(man1dir)/
+ $(INSTALL_PROGRAM) lz4$(EXT) $(DESTDIR)$(bindir)/lz4$(EXT)
+ $(LN_SF) lz4$(EXT) $(DESTDIR)$(bindir)/lz4c$(EXT)
+ $(LN_SF) lz4$(EXT) $(DESTDIR)$(bindir)/lz4cat$(EXT)
+ $(LN_SF) lz4$(EXT) $(DESTDIR)$(bindir)/unlz4$(EXT)
+ @echo Installing man pages in $(DESTDIR)$(man1dir)
+ $(INSTALL_DATA) lz4.1 $(DESTDIR)$(man1dir)/lz4.1
+ $(LN_SF) lz4.1 $(DESTDIR)$(man1dir)/lz4c.1
+ $(LN_SF) lz4.1 $(DESTDIR)$(man1dir)/lz4cat.1
+ $(LN_SF) lz4.1 $(DESTDIR)$(man1dir)/unlz4.1
@echo lz4 installation completed
uninstall:
- @$(RM) $(DESTDIR)$(bindir)/lz4cat$(EXT)
- @$(RM) $(DESTDIR)$(bindir)/unlz4$(EXT)
- @$(RM) $(DESTDIR)$(bindir)/lz4$(EXT)
- @$(RM) $(DESTDIR)$(bindir)/lz4c$(EXT)
- @$(RM) $(DESTDIR)$(man1dir)/lz4.1
- @$(RM) $(DESTDIR)$(man1dir)/lz4c.1
- @$(RM) $(DESTDIR)$(man1dir)/lz4cat.1
- @$(RM) $(DESTDIR)$(man1dir)/unlz4.1
+ $(RM) $(DESTDIR)$(bindir)/lz4cat$(EXT)
+ $(RM) $(DESTDIR)$(bindir)/unlz4$(EXT)
+ $(RM) $(DESTDIR)$(bindir)/lz4$(EXT)
+ $(RM) $(DESTDIR)$(bindir)/lz4c$(EXT)
+ $(RM) $(DESTDIR)$(man1dir)/lz4.1
+ $(RM) $(DESTDIR)$(man1dir)/lz4c.1
+ $(RM) $(DESTDIR)$(man1dir)/lz4cat.1
+ $(RM) $(DESTDIR)$(man1dir)/unlz4.1
@echo lz4 programs successfully uninstalled
endif
diff --git a/programs/bench.c b/programs/bench.c
index 3357d14..4d35ef9 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -1,6 +1,6 @@
/*
bench.c - Demo program to benchmark open-source compression algorithms
- Copyright (C) Yann Collet 2012-2016
+ Copyright (C) Yann Collet 2012-2020
GPL v2 License
@@ -51,10 +51,101 @@
#include "lz4.h"
#define LZ4_HC_STATIC_LINKING_ONLY
#include "lz4hc.h"
+#include "lz4frame.h" /* LZ4F_decompress */
/* *************************************
-* Compression parameters and functions
+* Constants
+***************************************/
+#ifndef LZ4_GIT_COMMIT_STRING
+# define LZ4_GIT_COMMIT_STRING ""
+#else
+# define LZ4_GIT_COMMIT_STRING LZ4_EXPAND_AND_QUOTE(LZ4_GIT_COMMIT)
+#endif
+
+#define NBSECONDS 3
+#define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */
+#define TIMELOOP_NANOSEC 1*1000000000ULL /* 1 second */
+#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */
+#define COOLPERIOD_SEC 10
+#define DECOMP_MULT 1 /* test decompression DECOMP_MULT times longer than compression */
+
+#define KB *(1 <<10)
+#define MB *(1 <<20)
+#define GB *(1U<<30)
+
+#define LZ4_MAX_DICT_SIZE (64 KB)
+
+static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
+
+static U32 g_compressibilityDefault = 50;
+
+
+/* *************************************
+* console display
+***************************************/
+#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
+
+#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
+ if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \
+ { g_time = clock(); DISPLAY(__VA_ARGS__); \
+ if (g_displayLevel>=4) fflush(stdout); } }
+static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
+static clock_t g_time = 0;
+
+
+/* *************************************
+* DEBUG and error conditions
+***************************************/
+#ifndef DEBUG
+# define DEBUG 0
+#endif
+#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
+#define END_PROCESS(error, ...) \
+{ \
+ DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
+ DISPLAYLEVEL(1, "Error %i : ", error); \
+ DISPLAYLEVEL(1, __VA_ARGS__); \
+ DISPLAYLEVEL(1, "\n"); \
+ exit(error); \
+}
+
+#define LZ4_isError(errcode) (errcode==0)
+
+
+/* *************************************
+* Benchmark Parameters
+***************************************/
+static U32 g_nbSeconds = NBSECONDS;
+static size_t g_blockSize = 0;
+int g_additionalParam = 0;
+int g_benchSeparately = 0;
+int g_decodeOnly = 0;
+unsigned g_skipChecksums = 0;
+
+void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
+
+void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
+
+void BMK_setNbSeconds(unsigned nbSeconds)
+{
+ g_nbSeconds = nbSeconds;
+ DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbSeconds);
+}
+
+void BMK_setBlockSize(size_t blockSize) { g_blockSize = blockSize; }
+
+void BMK_setBenchSeparately(int separate) { g_benchSeparately = (separate!=0); }
+
+void BMK_setDecodeOnlyMode(int set) { g_decodeOnly = (set!=0); }
+
+void BMK_skipChecksums(int skip) { g_skipChecksums = (skip!=0); }
+
+
+/* *************************************
+ * Compression state management
***************************************/
struct compressionParameters
@@ -79,8 +170,8 @@ struct compressionParameters
const struct compressionParameters* pThis);
};
-static void LZ4_compressInitNoStream(
- struct compressionParameters* pThis)
+static void
+LZ4_compressInitNoStream(struct compressionParameters* pThis)
{
pThis->LZ4_stream = NULL;
pThis->LZ4_dictStream = NULL;
@@ -88,8 +179,8 @@ static void LZ4_compressInitNoStream(
pThis->LZ4_dictStreamHC = NULL;
}
-static void LZ4_compressInitStream(
- struct compressionParameters* pThis)
+static void
+LZ4_compressInitStream(struct compressionParameters* pThis)
{
pThis->LZ4_stream = LZ4_createStream();
pThis->LZ4_dictStream = LZ4_createStream();
@@ -98,8 +189,8 @@ static void LZ4_compressInitStream(
LZ4_loadDict(pThis->LZ4_dictStream, pThis->dictBuf, pThis->dictSize);
}
-static void LZ4_compressInitStreamHC(
- struct compressionParameters* pThis)
+static void
+LZ4_compressInitStreamHC(struct compressionParameters* pThis)
{
pThis->LZ4_stream = NULL;
pThis->LZ4_dictStream = NULL;
@@ -108,83 +199,84 @@ static void LZ4_compressInitStreamHC(
LZ4_loadDictHC(pThis->LZ4_dictStreamHC, pThis->dictBuf, pThis->dictSize);
}
-static void LZ4_compressResetNoStream(
- const struct compressionParameters* pThis)
+static void
+LZ4_compressResetNoStream(const struct compressionParameters* pThis)
{
(void)pThis;
}
-static void LZ4_compressResetStream(
- const struct compressionParameters* pThis)
+static void
+LZ4_compressResetStream(const struct compressionParameters* pThis)
{
LZ4_resetStream_fast(pThis->LZ4_stream);
LZ4_attach_dictionary(pThis->LZ4_stream, pThis->LZ4_dictStream);
}
-static void LZ4_compressResetStreamHC(
- const struct compressionParameters* pThis)
+static void
+LZ4_compressResetStreamHC(const struct compressionParameters* pThis)
{
LZ4_resetStreamHC_fast(pThis->LZ4_streamHC, pThis->cLevel);
LZ4_attach_HC_dictionary(pThis->LZ4_streamHC, pThis->LZ4_dictStreamHC);
}
-static int LZ4_compressBlockNoStream(
- const struct compressionParameters* pThis,
- const char* src, char* dst,
- int srcSize, int dstSize)
+static int
+LZ4_compressBlockNoStream(const struct compressionParameters* pThis,
+ const char* src, char* dst,
+ int srcSize, int dstSize)
{
int const acceleration = (pThis->cLevel < 0) ? -pThis->cLevel + 1 : 1;
return LZ4_compress_fast(src, dst, srcSize, dstSize, acceleration);
}
-static int LZ4_compressBlockNoStreamHC(
- const struct compressionParameters* pThis,
- const char* src, char* dst,
- int srcSize, int dstSize)
+static int
+LZ4_compressBlockNoStreamHC(const struct compressionParameters* pThis,
+ const char* src, char* dst,
+ int srcSize, int dstSize)
{
return LZ4_compress_HC(src, dst, srcSize, dstSize, pThis->cLevel);
}
-static int LZ4_compressBlockStream(
- const struct compressionParameters* pThis,
- const char* src, char* dst,
- int srcSize, int dstSize)
+static int
+LZ4_compressBlockStream(const struct compressionParameters* pThis,
+ const char* src, char* dst,
+ int srcSize, int dstSize)
{
int const acceleration = (pThis->cLevel < 0) ? -pThis->cLevel + 1 : 1;
return LZ4_compress_fast_continue(pThis->LZ4_stream, src, dst, srcSize, dstSize, acceleration);
}
-static int LZ4_compressBlockStreamHC(
- const struct compressionParameters* pThis,
- const char* src, char* dst,
- int srcSize, int dstSize)
+static int
+LZ4_compressBlockStreamHC(const struct compressionParameters* pThis,
+ const char* src, char* dst,
+ int srcSize, int dstSize)
{
return LZ4_compress_HC_continue(pThis->LZ4_streamHC, src, dst, srcSize, dstSize);
}
-static void LZ4_compressCleanupNoStream(
- const struct compressionParameters* pThis)
+static void
+LZ4_compressCleanupNoStream(const struct compressionParameters* pThis)
{
(void)pThis;
}
-static void LZ4_compressCleanupStream(
- const struct compressionParameters* pThis)
+static void
+LZ4_compressCleanupStream(const struct compressionParameters* pThis)
{
LZ4_freeStream(pThis->LZ4_stream);
LZ4_freeStream(pThis->LZ4_dictStream);
}
-static void LZ4_compressCleanupStreamHC(
- const struct compressionParameters* pThis)
+static void
+LZ4_compressCleanupStreamHC(const struct compressionParameters* pThis)
{
LZ4_freeStreamHC(pThis->LZ4_streamHC);
LZ4_freeStreamHC(pThis->LZ4_dictStreamHC);
}
-static void LZ4_buildCompressionParameters(
- struct compressionParameters* pParams,
- int cLevel, const char* dictBuf, int dictSize)
+static void
+LZ4_buildCompressionParameters(struct compressionParameters* pParams,
+ int cLevel,
+ const char* dictBuf, int dictSize)
{
pParams->cLevel = cLevel;
pParams->dictBuf = dictBuf;
@@ -215,90 +307,35 @@ static void LZ4_buildCompressionParameters(
}
}
-#define LZ4_isError(errcode) (errcode==0)
+typedef int (*DecFunction_f)(const char* src, char* dst,
+ int srcSize, int dstCapacity,
+ const char* dictStart, int dictSize);
-/* *************************************
-* Constants
-***************************************/
-#ifndef LZ4_GIT_COMMIT_STRING
-# define LZ4_GIT_COMMIT_STRING ""
-#else
-# define LZ4_GIT_COMMIT_STRING LZ4_EXPAND_AND_QUOTE(LZ4_GIT_COMMIT)
-#endif
-
-#define NBSECONDS 3
-#define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */
-#define TIMELOOP_NANOSEC 1*1000000000ULL /* 1 second */
-#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */
-#define COOLPERIOD_SEC 10
-#define DECOMP_MULT 1 /* test decompression DECOMP_MULT times longer than compression */
-
-#define KB *(1 <<10)
-#define MB *(1 <<20)
-#define GB *(1U<<30)
-
-#define LZ4_MAX_DICT_SIZE (64 KB)
-
-static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
-
-static U32 g_compressibilityDefault = 50;
-
-
-/* *************************************
-* console display
-***************************************/
-#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
-#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
-static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
-
-#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
- if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \
- { g_time = clock(); DISPLAY(__VA_ARGS__); \
- if (g_displayLevel>=4) fflush(stdout); } }
-static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
-static clock_t g_time = 0;
-
-
-/* *************************************
-* Exceptions
-***************************************/
-#ifndef DEBUG
-# define DEBUG 0
-#endif
-#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
-#define EXM_THROW(error, ...) \
-{ \
- DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
- DISPLAYLEVEL(1, "Error %i : ", error); \
- DISPLAYLEVEL(1, __VA_ARGS__); \
- DISPLAYLEVEL(1, "\n"); \
- exit(error); \
-}
-
+static LZ4F_dctx* g_dctx = NULL;
-/* *************************************
-* Benchmark Parameters
-***************************************/
-static U32 g_nbSeconds = NBSECONDS;
-static size_t g_blockSize = 0;
-int g_additionalParam = 0;
-int g_benchSeparately = 0;
-
-void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
-
-void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
-
-void BMK_setNbSeconds(unsigned nbSeconds)
+static int
+LZ4F_decompress_binding(const char* src, char* dst,
+ int srcSize, int dstCapacity,
+ const char* dictStart, int dictSize)
{
- g_nbSeconds = nbSeconds;
- DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbSeconds);
+ size_t dstSize = (size_t)dstCapacity;
+ size_t readSize = (size_t)srcSize;
+ LZ4F_decompressOptions_t dOpt = { 1, 0, 0, 0 };
+ size_t decStatus;
+ dOpt.skipChecksums = g_skipChecksums;
+ decStatus = LZ4F_decompress(g_dctx,
+ dst, &dstSize,
+ src, &readSize,
+ &dOpt);
+ if ( (decStatus == 0) /* decompression successful */
+ && ((int)readSize==srcSize) /* consume all input */ )
+ return (int)dstSize;
+ /* else, error */
+ return -1;
+ (void)dictStart; (void)dictSize; /* not compatible with dictionary yet */
}
-void BMK_setBlockSize(size_t blockSize) { g_blockSize = blockSize; }
-
-void BMK_setBenchSeparately(int separate) { g_benchSeparately = (separate!=0); }
-
/* ********************************************************
* Bench functions
@@ -321,24 +358,32 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
const size_t* fileSizes, U32 nbFiles,
const char* dictBuf, int dictSize)
{
- size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
- U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
+ size_t const blockSize = (g_blockSize>=32 && !g_decodeOnly ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
+ U32 const maxNbBlocks = (U32)((srcSize + (blockSize-1)) / blockSize) + nbFiles;
blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
- size_t const maxCompressedSize = LZ4_compressBound((int)srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
+ size_t const maxCompressedSize = (size_t)LZ4_compressBound((int)srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
void* const compressedBuffer = malloc(maxCompressedSize);
- void* const resultBuffer = malloc(srcSize);
+ size_t const decMultiplier = g_decodeOnly ? 255 : 1;
+ size_t const maxInSize = (size_t)LZ4_MAX_INPUT_SIZE / decMultiplier;
+ size_t const maxDecSize = srcSize < maxInSize ? srcSize * decMultiplier : LZ4_MAX_INPUT_SIZE;
+ void* const resultBuffer = malloc(maxDecSize);
U32 nbBlocks;
struct compressionParameters compP;
/* checks */
if (!compressedBuffer || !resultBuffer || !blockTable)
- EXM_THROW(31, "allocation error : not enough memory");
+ END_PROCESS(31, "allocation error : not enough memory");
if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */
/* init */
LZ4_buildCompressionParameters(&compP, cLevel, dictBuf, dictSize);
compP.initFunction(&compP);
+ if (g_dctx==NULL) {
+ LZ4F_createDecompressionContext(&g_dctx, LZ4F_VERSION);
+ if (g_dctx==NULL)
+ END_PROCESS(1, "allocation error - decompression state");
+ }
/* Init blockTable data */
{ const char* srcPtr = (const char*)srcBuffer;
@@ -351,6 +396,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
U32 const blockEnd = nbBlocks + nbBlocksforThisFile;
for ( ; nbBlocks<blockEnd; nbBlocks++) {
size_t const thisBlockSize = MIN(remaining, blockSize);
+ size_t const resMaxSize = thisBlockSize * decMultiplier;
+ size_t const resCapa = (thisBlockSize < maxInSize) ? resMaxSize : LZ4_MAX_INPUT_SIZE;
blockTable[nbBlocks].srcPtr = srcPtr;
blockTable[nbBlocks].cPtr = cPtr;
blockTable[nbBlocks].resPtr = resPtr;
@@ -358,29 +405,37 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
blockTable[nbBlocks].cRoom = (size_t)LZ4_compressBound((int)thisBlockSize);
srcPtr += thisBlockSize;
cPtr += blockTable[nbBlocks].cRoom;
- resPtr += thisBlockSize;
+ resPtr += resCapa;
remaining -= thisBlockSize;
} } }
- /* warmimg up memory */
+ /* warming up memory */
RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
+ /* decode-only mode : copy input to @compressedBuffer */
+ if (g_decodeOnly) {
+ U32 blockNb;
+ for (blockNb=0; blockNb < nbBlocks; blockNb++) {
+ memcpy(blockTable[blockNb].cPtr, blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize);
+ blockTable[blockNb].cSize = blockTable[blockNb].srcSize;
+ } }
+
/* Bench */
{ U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL);
U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
- UTIL_time_t coolTime;
+ UTIL_time_t coolTime = UTIL_getTime();
U64 const maxTime = (g_nbSeconds * TIMELOOP_NANOSEC) + 100;
U32 nbCompressionLoops = (U32)((5 MB) / (srcSize+1)) + 1; /* conservative initial compression speed estimate */
U32 nbDecodeLoops = (U32)((200 MB) / (srcSize+1)) + 1; /* conservative initial decode speed estimate */
U64 totalCTime=0, totalDTime=0;
- U32 cCompleted=0, dCompleted=0;
+ U32 cCompleted=(g_decodeOnly==1), dCompleted=0;
# define NB_MARKS 4
const char* const marks[NB_MARKS] = { " |", " /", " =", "\\" };
U32 markNb = 0;
- size_t cSize = 0;
+ size_t cSize = srcSize;
+ size_t totalRSize = srcSize;
double ratio = 0.;
- coolTime = UTIL_getTime();
DISPLAYLEVEL(2, "\r%79s\r", "");
while (!cCompleted || !dCompleted) {
/* overheat protection */
@@ -391,8 +446,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
}
/* Compression */
- DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
- if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */
+ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)totalRSize);
+ if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase compressed buffer */
UTIL_sleepMilli(1); /* give processor time to other processes */
UTIL_waitForNextTick();
@@ -408,7 +463,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
&compP,
blockTable[blockNb].srcPtr, blockTable[blockNb].cPtr,
(int)blockTable[blockNb].srcSize, (int)blockTable[blockNb].cRoom);
- if (LZ4_isError(rSize)) EXM_THROW(1, "LZ4 compression failed");
+ if (LZ4_isError(rSize)) END_PROCESS(1, "LZ4 compression failed");
blockTable[blockNb].cSize = rSize;
} }
{ U64 const clockSpan = UTIL_clockSpanNano(clockStart);
@@ -423,17 +478,18 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
}
totalCTime += clockSpan;
cCompleted = totalCTime>maxTime;
- } }
-
- cSize = 0;
- { U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
- cSize += !cSize; /* avoid div by 0 */
- ratio = (double)srcSize / (double)cSize;
- markNb = (markNb+1) % NB_MARKS;
- DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
- marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
- ((double)srcSize / fastestC) * 1000 );
-
+ }
+
+ cSize = 0;
+ { U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
+ cSize += !cSize; /* avoid div by 0 */
+ ratio = (double)totalRSize / (double)cSize;
+ markNb = (markNb+1) % NB_MARKS;
+ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
+ marks[markNb], displayName,
+ (U32)totalRSize, (U32)cSize, ratio,
+ ((double)totalRSize / fastestC) * 1000 );
+ }
(void)fastestD; (void)crcOrig; /* unused when decompression disabled */
#if 1
/* Decompression */
@@ -443,17 +499,30 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
UTIL_waitForNextTick();
if (!dCompleted) {
+ const DecFunction_f decFunction = g_decodeOnly ?
+ LZ4F_decompress_binding : LZ4_decompress_safe_usingDict;
+ const char* const decString = g_decodeOnly ?
+ "LZ4F_decompress" : "LZ4_decompress_safe_usingDict";
UTIL_time_t const clockStart = UTIL_getTime();
U32 nbLoops;
+
for (nbLoops=0; nbLoops < nbDecodeLoops; nbLoops++) {
U32 blockNb;
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
- int const regenSize = LZ4_decompress_safe_usingDict(
+ size_t const inMaxSize = (size_t)INT_MAX / decMultiplier;
+ size_t const resCapa = (blockTable[blockNb].srcSize < inMaxSize) ?
+ blockTable[blockNb].srcSize * decMultiplier :
+ INT_MAX;
+ int const regenSize = decFunction(
blockTable[blockNb].cPtr, blockTable[blockNb].resPtr,
- (int)blockTable[blockNb].cSize, (int)blockTable[blockNb].srcSize,
+ (int)blockTable[blockNb].cSize, (int)resCapa,
dictBuf, dictSize);
if (regenSize < 0) {
- DISPLAY("LZ4_decompress_safe_usingDict() failed on block %u \n", blockNb);
+ DISPLAY("%s() failed on block %u of size %u \n",
+ decString, blockNb, (unsigned)blockTable[blockNb].srcSize);
+ if (g_decodeOnly)
+ DISPLAY("Is input using LZ4 Frame format ? \n");
+ END_PROCESS(2, "error during decoding");
break;
}
blockTable[blockNb].resSize = (size_t)regenSize;
@@ -472,14 +541,22 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
dCompleted = totalDTime > (DECOMP_MULT*maxTime);
} }
+ if (g_decodeOnly) {
+ unsigned u;
+ totalRSize = 0;
+ for (u=0; u<nbBlocks; u++) totalRSize += blockTable[u].resSize;
+ }
markNb = (markNb+1) % NB_MARKS;
+ ratio = (double)totalRSize / (double)cSize;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
- marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
- ((double)srcSize / fastestC) * 1000,
- ((double)srcSize / fastestD) * 1000);
-
- /* CRC Checking */
- { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
+ marks[markNb], displayName,
+ (U32)totalRSize, (U32)cSize, ratio,
+ ((double)totalRSize / fastestC) * 1000,
+ ((double)totalRSize / fastestD) * 1000);
+
+ /* CRC Checking (not possible in decode-only mode)*/
+ if (!g_decodeOnly) {
+ U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
if (crcOrig!=crcCheck) {
size_t u;
DISPLAY("\n!!! WARNING !!! %17s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
@@ -594,21 +671,21 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
continue;
}
f = fopen(fileNamesTable[n], "rb");
- if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
+ if (f==NULL) END_PROCESS(10, "impossible to open file %s", fileNamesTable[n]);
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
if (fileSize > bufferSize-pos) { /* buffer too small - stop after this file */
fileSize = bufferSize-pos;
nbFiles=n;
}
{ size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
- if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
+ if (readSize != (size_t)fileSize) END_PROCESS(11, "could not read %s", fileNamesTable[n]);
pos += readSize; }
fileSizes[n] = (size_t)fileSize;
totalSize += (size_t)fileSize;
fclose(f);
}
- if (totalSize == 0) EXM_THROW(12, "no data to bench");
+ if (totalSize == 0) END_PROCESS(12, "no data to bench");
}
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
@@ -621,11 +698,11 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
char mfName[20] = {0};
- if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes");
+ if (!fileSizes) END_PROCESS(12, "not enough memory for fileSizes");
/* Memory allocation & restrictions */
benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3;
- if (benchedSize==0) EXM_THROW(12, "not enough memory");
+ if (benchedSize==0) END_PROCESS(12, "not enough memory");
if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
if (benchedSize > LZ4_MAX_INPUT_SIZE) {
benchedSize = LZ4_MAX_INPUT_SIZE;
@@ -635,7 +712,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
}
srcBuffer = malloc(benchedSize + !benchedSize); /* avoid alloc of zero */
- if (!srcBuffer) EXM_THROW(12, "not enough memory");
+ if (!srcBuffer) END_PROCESS(12, "not enough memory");
/* Load input buffer */
BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
@@ -663,7 +740,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
void* const srcBuffer = malloc(benchedSize);
/* Memory allocation */
- if (!srcBuffer) EXM_THROW(21, "not enough memory");
+ if (!srcBuffer) END_PROCESS(21, "not enough memory");
/* Fill input buffer */
RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
@@ -677,7 +754,8 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
}
-int BMK_benchFilesSeparately(const char** fileNamesTable, unsigned nbFiles,
+static int
+BMK_benchFilesSeparately(const char** fileNamesTable, unsigned nbFiles,
int cLevel, int cLevelLast,
const char* dictBuf, int dictSize)
{
@@ -685,7 +763,6 @@ int BMK_benchFilesSeparately(const char** fileNamesTable, unsigned nbFiles,
if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX;
if (cLevelLast > LZ4HC_CLEVEL_MAX) cLevelLast = LZ4HC_CLEVEL_MAX;
if (cLevelLast < cLevel) cLevelLast = cLevel;
- if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
for (fileNb=0; fileNb<nbFiles; fileNb++)
BMK_benchFileTable(fileNamesTable+fileNb, 1, cLevel, cLevelLast, dictBuf, dictSize);
@@ -700,45 +777,59 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
{
double const compressibility = (double)g_compressibilityDefault / 100;
char* dictBuf = NULL;
- int dictSize = 0;
+ size_t dictSize = 0;
if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX;
+ if (g_decodeOnly) {
+ DISPLAYLEVEL(2, "Benchmark Decompression of LZ4 Frame ");
+ if (g_skipChecksums) {
+ DISPLAYLEVEL(2, "_without_ checksum even when present \n");
+ } else {
+ DISPLAYLEVEL(2, "+ Checksum when present \n");
+ }
+ cLevelLast = cLevel;
+ }
if (cLevelLast > LZ4HC_CLEVEL_MAX) cLevelLast = LZ4HC_CLEVEL_MAX;
if (cLevelLast < cLevel) cLevelLast = cLevel;
- if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
+ if (cLevelLast > cLevel)
+ DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
if (dictFileName) {
FILE* dictFile = NULL;
- U64 dictFileSize = UTIL_getFileSize(dictFileName);
- if (!dictFileSize) EXM_THROW(25, "Dictionary error : could not stat dictionary file");
+ U64 const dictFileSize = UTIL_getFileSize(dictFileName);
+ if (!dictFileSize)
+ END_PROCESS(25, "Dictionary error : could not stat dictionary file");
+ if (g_decodeOnly)
+ END_PROCESS(26, "Error : LZ4 Frame decoder mode not compatible with dictionary yet");
dictFile = fopen(dictFileName, "rb");
- if (!dictFile) EXM_THROW(25, "Dictionary error : could not open dictionary file");
+ if (!dictFile)
+ END_PROCESS(25, "Dictionary error : could not open dictionary file");
if (dictFileSize > LZ4_MAX_DICT_SIZE) {
dictSize = LZ4_MAX_DICT_SIZE;
- if (UTIL_fseek(dictFile, dictFileSize - dictSize, SEEK_SET))
- EXM_THROW(25, "Dictionary error : could not seek dictionary file");
+ if (UTIL_fseek(dictFile, (long)(dictFileSize - dictSize), SEEK_SET))
+ END_PROCESS(25, "Dictionary error : could not seek dictionary file");
} else {
- dictSize = (int)dictFileSize;
+ dictSize = (size_t)dictFileSize;
}
- dictBuf = (char *)malloc(dictSize);
- if (!dictBuf) EXM_THROW(25, "Allocation error : not enough memory");
+ dictBuf = (char*)malloc(dictSize);
+ if (!dictBuf) END_PROCESS(25, "Allocation error : not enough memory");
- if (fread(dictBuf, 1, dictSize, dictFile) != (size_t)dictSize)
- EXM_THROW(25, "Dictionary error : could not read dictionary file");
+ if (fread(dictBuf, 1, dictSize, dictFile) != dictSize)
+ END_PROCESS(25, "Dictionary error : could not read dictionary file");
fclose(dictFile);
}
if (nbFiles == 0)
- BMK_syntheticTest(cLevel, cLevelLast, compressibility, dictBuf, dictSize);
+ BMK_syntheticTest(cLevel, cLevelLast, compressibility, dictBuf, (int)dictSize);
else {
if (g_benchSeparately)
- BMK_benchFilesSeparately(fileNamesTable, nbFiles, cLevel, cLevelLast, dictBuf, dictSize);
+ BMK_benchFilesSeparately(fileNamesTable, nbFiles, cLevel, cLevelLast, dictBuf, (int)dictSize);
else
- BMK_benchFileTable(fileNamesTable, nbFiles, cLevel, cLevelLast, dictBuf, dictSize);
+ BMK_benchFileTable(fileNamesTable, nbFiles, cLevel, cLevelLast, dictBuf, (int)dictSize);
}
free(dictBuf);
diff --git a/programs/bench.h b/programs/bench.h
index 22ebf60..1d81a99 100644
--- a/programs/bench.h
+++ b/programs/bench.h
@@ -1,6 +1,6 @@
/*
bench.h - Demo program to benchmark open-source compression algorithm
- Copyright (C) Yann Collet 2012-2016
+ Copyright (C) Yann Collet 2012-2020
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -25,15 +25,30 @@
#include <stddef.h>
+/* BMK_benchFiles() :
+ * Benchmark all files provided through array @fileNamesTable.
+ * All files must be valid, otherwise benchmark fails.
+ * Roundtrip measurements are done for each file individually, but
+ * unless BMK_setBenchSeparately() is set, all results are agglomerated.
+ * The method benchmarks all compression levels from @cLevelStart to @cLevelLast,
+ * both inclusive, providing one result per compression level.
+ * If @cLevelLast <= @cLevelStart, BMK_benchFiles() benchmarks @cLevelStart only.
+ * @dictFileName is optional, it's possible to provide NULL.
+ * When provided, compression and decompression use the specified file as dictionary.
+ * Only one dictionary can be provided, in which case it's applied to all benchmarked files.
+**/
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
- int cLevel, int cLevelLast,
+ int cLevelStart, int cLevelLast,
const char* dictFileName);
/* Set Parameters */
-void BMK_setNbSeconds(unsigned nbLoops);
-void BMK_setBlockSize(size_t blockSize);
-void BMK_setAdditionalParam(int additionalParam);
-void BMK_setNotificationLevel(unsigned level);
-void BMK_setBenchSeparately(int separate);
+void BMK_setNbSeconds(unsigned nbSeconds); /* minimum benchmark duration, in seconds, for both compression and decompression */
+void BMK_setBlockSize(size_t blockSize); /* Internally cut input file(s) into independent blocks of specified size */
+void BMK_setNotificationLevel(unsigned level); /* Influence verbosity level */
+void BMK_setBenchSeparately(int separate); /* When providing multiple files, output one result per file */
+void BMK_setDecodeOnlyMode(int set); /* v1.9.4+: set benchmark mode to decode only */
+void BMK_skipChecksums(int skip); /* v1.9.4+: only useful for DecodeOnlyMode; do not calculate checksum when present, to save CPU time */
+
+void BMK_setAdditionalParam(int additionalParam); /* hidden param, influence output format, for python parsing */
#endif /* BENCH_H_125623623633 */
diff --git a/programs/datagen.c b/programs/datagen.c
index 24a2da2..f448640 100644
--- a/programs/datagen.c
+++ b/programs/datagen.c
@@ -1,6 +1,6 @@
/*
datagen.c - compressible data generator test tool
- Copyright (C) Yann Collet 2012-2016
+ Copyright (C) Yann Collet 2012-2020
GPL v2 License
diff --git a/programs/datagen.h b/programs/datagen.h
index 91c5b02..c20c9c7 100644
--- a/programs/datagen.h
+++ b/programs/datagen.h
@@ -1,6 +1,6 @@
/*
datagen.h - compressible data generator header
- Copyright (C) Yann Collet 2012-2016
+ Copyright (C) Yann Collet 2012-2020
GPL v2 License
diff --git a/programs/lz4-exe.rc.in b/programs/lz4-exe.rc.in
index 7b81030..bcf4d7d 100644
--- a/programs/lz4-exe.rc.in
+++ b/programs/lz4-exe.rc.in
@@ -13,7 +13,7 @@ FILETYPE 1
VALUE "FileDescription", "Extremely fast compression"
VALUE "FileVersion", "@LIBVER_MAJOR@.@LIBVER_MINOR@.@LIBVER_PATCH@.0"
VALUE "InternalName", "@PROGNAME@"
- VALUE "LegalCopyright", "Copyright (C) 2013-2016, Yann Collet"
+ VALUE "LegalCopyright", "Copyright (C) 2013-2020, Yann Collet"
VALUE "OriginalFilename", "@PROGNAME@.@EXT@"
VALUE "ProductName", "LZ4"
VALUE "ProductVersion", "@LIBVER_MAJOR@.@LIBVER_MINOR@.@LIBVER_PATCH@.0"
@@ -24,4 +24,3 @@ FILETYPE 1
VALUE "Translation", 0x0409, 1200
}
}
-
diff --git a/programs/lz4.1 b/programs/lz4.1
index d758ed5..7cb98d6 100644
--- a/programs/lz4.1
+++ b/programs/lz4.1
@@ -1,5 +1,5 @@
.
-.TH "LZ4" "1" "July 2019" "lz4 1.9.2" "User Commands"
+.TH "LZ4" "1" "August 2022" "lz4 v1.9.4" "User Commands"
.
.SH "NAME"
\fBlz4\fR \- lz4, unlz4, lz4cat \- Compress or decompress \.lz4 files
@@ -17,7 +17,7 @@
When writing scripts that need to decompress files, it is recommended to always use the name \fBlz4\fR with appropriate arguments (\fBlz4 \-d\fR or \fBlz4 \-dc\fR) instead of the names \fBunlz4\fR and \fBlz4cat\fR\.
.
.SH "DESCRIPTION"
-\fBlz4\fR is an extremely fast lossless compression algorithm, based on \fBbyte\-aligned LZ77\fR family of compression scheme\. \fBlz4\fR offers compression speeds of 400 MB/s per core, linearly scalable with multi\-core CPUs\. It features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limit on multi\-core systems\. The native file format is the \fB\.lz4\fR format\.
+\fBlz4\fR is an extremely fast lossless compression algorithm, based on \fBbyte\-aligned LZ77\fR family of compression scheme\. \fBlz4\fR offers compression speeds > 500 MB/s per core, linearly scalable with multi\-core CPUs\. It features an extremely fast decoder, offering speed in multiple GB/s per core, typically reaching RAM speed limit on multi\-core systems\. The native file format is the \fB\.lz4\fR format\.
.
.SS "Difference between lz4 and gzip"
\fBlz4\fR supports a command line syntax similar \fIbut not identical\fR to \fBgzip(1)\fR\. Differences are :
@@ -32,7 +32,7 @@ When writing scripts that need to decompress files, it is recommended to always
\fBlz4 file\.lz4\fR will default to decompression (use \fB\-z\fR to force compression)
.
.IP "\(bu" 4
-\fBlz4\fR preserves original files
+\fBlz4\fR preserves original files (see \fB\-\-rm\fR to erase source file on completion)
.
.IP "\(bu" 4
\fBlz4\fR shows real\-time notification statistics during compression or decompression of a single file (use \fB\-q\fR to silence them)
@@ -121,7 +121,7 @@ Switch to ultra\-fast compression levels\. The higher the value, the faster the
.
.TP
\fB\-\-best\fR
-Set highest compression level\. Same as -12\.
+Set highest compression level\. Same as \-12\.
.
.TP
\fB\-\-favor\-decSpeed\fR
@@ -169,10 +169,18 @@ Produce independent blocks (default)
Blocks depend on predecessors (improves compression ratio, more noticeable on small blocks)
.
.TP
+\fB\-BX\fR
+Generate block checksums (default:disabled)
+.
+.TP
\fB\-\-[no\-]frame\-crc\fR
Select frame checksum (default:enabled)
.
.TP
+\fB\-\-no\-crc\fR
+Disable both frame and block checksums
+.
+.TP
\fB\-\-[no\-]content\-size\fR
Header includes original size (default:not present)
.
diff --git a/programs/lz4.1.md b/programs/lz4.1.md
index 56c0053..06c06cf 100644
--- a/programs/lz4.1.md
+++ b/programs/lz4.1.md
@@ -20,9 +20,9 @@ DESCRIPTION
`lz4` is an extremely fast lossless compression algorithm,
based on **byte-aligned LZ77** family of compression scheme.
-`lz4` offers compression speeds of 400 MB/s per core, linearly scalable with
-multi-core CPUs.
-It features an extremely fast decoder, with speed in multiple GB/s per core,
+`lz4` offers compression speeds > 500 MB/s per core,
+linearly scalable with multi-core CPUs.
+It features an extremely fast decoder, offering speed in multiple GB/s per core,
typically reaching RAM speed limit on multi-core systems.
The native file format is the `.lz4` format.
@@ -34,7 +34,7 @@ Differences are :
* `lz4` compresses a single file by default (see `-m` for multiple files)
* `lz4 file1 file2` means : compress file1 _into_ file2
* `lz4 file.lz4` will default to decompression (use `-z` to force compression)
- * `lz4` preserves original files
+ * `lz4` preserves original files (see `--rm` to erase source file on completion)
* `lz4` shows real-time notification statistics
during compression or decompression of a single file
(use `-q` to silence them)
@@ -185,9 +185,15 @@ only the latest one will be applied.
* `-BD`:
Blocks depend on predecessors (improves compression ratio, more noticeable on small blocks)
+* `-BX`:
+ Generate block checksums (default:disabled)
+
* `--[no-]frame-crc`:
Select frame checksum (default:enabled)
+* `--no-crc`:
+ Disable both frame and block checksums
+
* `--[no-]content-size`:
Header includes original size (default:not present)<br/>
Note : this option can only be activated when the original size can be
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index 523b8a8..8c3f9fd 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -1,6 +1,6 @@
/*
LZ4cli - LZ4 Command Line Interface
- Copyright (C) Yann Collet 2011-2016
+ Copyright (C) Yann Collet 2011-2020
GPL v2 License
@@ -186,7 +186,7 @@ static int usage_longhelp(const char* exeName)
DISPLAY( "\n");
DISPLAY( "Compression levels : \n");
DISPLAY( "---------------------\n");
- DISPLAY( "-0 ... -2 => Fast compression, all identicals\n");
+ DISPLAY( "-0 ... -2 => Fast compression, all identical\n");
DISPLAY( "-3 ... -%d => High compression; higher number == more compression but slower\n", LZ4HC_CLEVEL_MAX);
DISPLAY( "\n");
DISPLAY( "stdin, stdout and the console : \n");
@@ -314,6 +314,7 @@ int main(int argc, const char** argv)
cLevelLast=-10000,
legacy_format=0,
forceStdout=0,
+ forceOverwrite=0,
main_pause=0,
multiple_inputs=0,
all_arguments_are_files=0,
@@ -330,9 +331,8 @@ int main(int argc, const char** argv)
const char extension[] = LZ4_EXTENSION;
size_t blockSize = LZ4IO_setBlockSizeID(prefs, LZ4_BLOCKSIZEID_DEFAULT);
const char* const exeName = lastNameFromPath(argv[0]);
-#ifdef UTIL_HAS_CREATEFILELIST
- const char** extendedFileList = NULL;
char* fileNamesBuf = NULL;
+#ifdef UTIL_HAS_CREATEFILELIST
unsigned fileNamesNb, recursive=0;
#endif
@@ -377,16 +377,21 @@ int main(int argc, const char** argv)
if (argument[1]=='-') {
if (!strcmp(argument, "--")) { all_arguments_are_files = 1; continue; }
if (!strcmp(argument, "--compress")) { mode = om_compress; continue; }
- if ((!strcmp(argument, "--decompress"))
- || (!strcmp(argument, "--uncompress"))) { mode = om_decompress; continue; }
+ if ( (!strcmp(argument, "--decompress"))
+ || (!strcmp(argument, "--uncompress"))) {
+ if (mode != om_bench) mode = om_decompress;
+ BMK_setDecodeOnlyMode(1);
+ continue;
+ }
if (!strcmp(argument, "--multiple")) { multiple_inputs = 1; continue; }
if (!strcmp(argument, "--test")) { mode = om_test; continue; }
if (!strcmp(argument, "--force")) { LZ4IO_setOverwrite(prefs, 1); continue; }
if (!strcmp(argument, "--no-force")) { LZ4IO_setOverwrite(prefs, 0); continue; }
if ((!strcmp(argument, "--stdout"))
|| (!strcmp(argument, "--to-stdout"))) { forceStdout=1; output_filename=stdoutmark; continue; }
- if (!strcmp(argument, "--frame-crc")) { LZ4IO_setStreamChecksumMode(prefs, 1); continue; }
- if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(prefs, 0); continue; }
+ if (!strcmp(argument, "--frame-crc")) { LZ4IO_setStreamChecksumMode(prefs, 1); BMK_skipChecksums(0); continue; }
+ if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(prefs, 0); BMK_skipChecksums(1); continue; }
+ if (!strcmp(argument, "--no-crc")) { LZ4IO_setStreamChecksumMode(prefs, 0); LZ4IO_setBlockChecksumMode(prefs, 0); BMK_skipChecksums(1); continue; }
if (!strcmp(argument, "--content-size")) { LZ4IO_setContentSize(prefs, 1); continue; }
if (!strcmp(argument, "--no-content-size")) { LZ4IO_setContentSize(prefs, 0); continue; }
if (!strcmp(argument, "--list")) { mode = om_list; continue; }
@@ -478,7 +483,10 @@ int main(int argc, const char** argv)
case 'l': legacy_format = 1; blockSize = 8 MB; break;
/* Decoding */
- case 'd': mode = om_decompress; break;
+ case 'd':
+ if (mode != om_bench) mode = om_decompress;
+ BMK_setDecodeOnlyMode(1);
+ break;
/* Force stdout, even if stdout==console */
case 'c':
@@ -491,7 +499,7 @@ int main(int argc, const char** argv)
case 't': mode = om_test; break;
/* Overwrite */
- case 'f': LZ4IO_setOverwrite(prefs, 1); break;
+ case 'f': forceOverwrite=1; LZ4IO_setOverwrite(prefs, 1); break;
/* Verbose mode */
case 'v': displayLevel++; break;
@@ -581,20 +589,24 @@ int main(int argc, const char** argv)
}
/* Store in *inFileNames[] if -m is used. */
- if (multiple_inputs) { inFileNames[ifnIdx++]=argument; continue; }
+ if (multiple_inputs) { inFileNames[ifnIdx++] = argument; continue; }
- /* Store first non-option arg in input_filename to preserve original cli logic. */
- if (!input_filename) { input_filename=argument; continue; }
+ /* original cli logic : lz4 input output */
+ /* First non-option arg is input_filename. */
+ if (!input_filename) { input_filename = argument; continue; }
- /* Second non-option arg in output_filename to preserve original cli logic. */
+ /* Second non-option arg is output_filename */
if (!output_filename) {
- output_filename=argument;
+ output_filename = argument;
if (!strcmp (output_filename, nullOutput)) output_filename = nulmark;
continue;
}
- /* 3rd non-option arg should not exist */
- DISPLAYLEVEL(1, "Warning : %s won't be used ! Do you want multiple input files (-m) ? \n", argument);
+ /* 3rd+ non-option arg should not exist */
+ DISPLAYLEVEL(1, "%s : %s won't be used ! Do you want multiple input files (-m) ? \n",
+ forceOverwrite ? "Warning" : "Error",
+ argument);
+ if (!forceOverwrite) exit(1);
}
DISPLAYLEVEL(3, WELCOME_MESSAGE);
@@ -617,7 +629,7 @@ int main(int argc, const char** argv)
input_filename = inFileNames[0];
#ifdef UTIL_HAS_CREATEFILELIST
if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */
- extendedFileList = UTIL_createFileList(inFileNames, ifnIdx, &fileNamesBuf, &fileNamesNb);
+ const char** extendedFileList = UTIL_createFileList(inFileNames, ifnIdx, &fileNamesBuf, &fileNamesNb);
if (extendedFileList) {
unsigned u;
for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, extendedFileList[u]);
@@ -649,27 +661,19 @@ int main(int argc, const char** argv)
mode = om_decompress; /* defer to decompress */
}
- /* compress or decompress */
+ /* No input provided => use stdin */
if (!input_filename) input_filename = stdinmark;
- /* Check if input is defined as console; trigger an error in this case */
+
+ /* Refuse to use the console as input */
if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) {
DISPLAYLEVEL(1, "refusing to read from a console\n");
exit(1);
}
+
if (!strcmp(input_filename, stdinmark)) {
/* if input==stdin and no output defined, stdout becomes default output */
if (!output_filename) output_filename = stdoutmark;
}
- else{
-#ifdef UTIL_HAS_CREATEFILELIST
- if (!recursive && !UTIL_isRegFile(input_filename)) {
-#else
- if (!UTIL_isRegFile(input_filename)) {
-#endif
- DISPLAYLEVEL(1, "%s: is not a regular file \n", input_filename);
- exit(1);
- }
- }
/* No output filename ==> try to select one automatically (when possible) */
while ((!output_filename) && (multiple_inputs==0)) {
@@ -679,7 +683,7 @@ int main(int argc, const char** argv)
* To ensure `stdout` is explicitly selected, use `-c` command flag.
* Conversely, to ensure output will not become `stdout`, use `-m` command flag */
DISPLAYLEVEL(1, "Warning : using stdout as default output. Do not rely on this behavior: use explicit `-c` instead ! \n");
- output_filename=stdoutmark;
+ output_filename = stdoutmark;
break;
}
if (mode == om_auto) { /* auto-determine compression or decompression, based on file extension */
@@ -695,7 +699,7 @@ int main(int argc, const char** argv)
DISPLAYLEVEL(2, "Compressed filename will be : %s \n", output_filename);
break;
}
- if (mode == om_decompress) {/* decompression to file (automatic name will work only if input filename has correct format extension) */
+ if (mode == om_decompress) {/* decompress to file (automatic output name only works if input filename has correct format extension) */
size_t outl;
size_t const inl = strlen(input_filename);
dynNameSpace = (char*)calloc(1,inl+1);
@@ -704,32 +708,27 @@ int main(int argc, const char** argv)
outl = inl;
if (inl>4)
while ((outl >= inl-4) && (input_filename[outl] == extension[outl-inl+4])) dynNameSpace[outl--]=0;
- if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); badusage(exeName); }
+ if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename \n"); badusage(exeName); }
output_filename = dynNameSpace;
DISPLAYLEVEL(2, "Decoding file %s \n", output_filename);
}
break;
}
- if (mode == om_list){
- /* Exit if trying to read from stdin as this isn't supported in this mode */
- if(!strcmp(input_filename, stdinmark)){
- DISPLAYLEVEL(1, "refusing to read from standard input in --list mode\n");
- exit(1);
- }
- if(!multiple_inputs){
- inFileNames[ifnIdx++] = input_filename;
- }
- }
- else{
- if (multiple_inputs==0) assert(output_filename);
+ if (mode == om_list) {
+ if (!multiple_inputs) inFileNames[ifnIdx++] = input_filename;
+ } else {
+ if (!multiple_inputs) assert(output_filename != NULL);
}
/* when multiple_inputs==1, output_filename may simply be useless,
* however, output_filename must be !NULL for next strcmp() tests */
if (!output_filename) output_filename = "*\\dummy^!//";
/* Check if output is defined as console; trigger an error in this case */
- if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) {
+ if ( !strcmp(output_filename,stdoutmark)
+ && mode != om_list
+ && IS_CONSOLE(stdout)
+ && !forceStdout) {
DISPLAYLEVEL(1, "refusing to write to console without -c \n");
exit(1);
}
@@ -747,8 +746,10 @@ int main(int argc, const char** argv)
if (ifnIdx == 0) multiple_inputs = 0;
if (mode == om_decompress) {
if (multiple_inputs) {
- const char* const dec_extension = !strcmp(output_filename,stdoutmark) ? stdoutmark : LZ4_EXTENSION;
- assert(ifnIdx <= INT_MAX);
+ const char* dec_extension = LZ4_EXTENSION;
+ if (!strcmp(output_filename, stdoutmark)) dec_extension = stdoutmark;
+ if (!strcmp(output_filename, nulmark)) dec_extension = nulmark;
+ assert(ifnIdx < INT_MAX);
operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, (int)ifnIdx, dec_extension, prefs);
} else {
operationResult = DEFAULT_DECOMPRESSOR(input_filename, output_filename, prefs);
@@ -776,12 +777,7 @@ int main(int argc, const char** argv)
_cleanup:
if (main_pause) waitEnter();
free(dynNameSpace);
-#ifdef UTIL_HAS_CREATEFILELIST
- if (extendedFileList) {
- UTIL_freeFileList(extendedFileList, fileNamesBuf);
- inFileNames = NULL;
- }
-#endif
+ free(fileNamesBuf);
LZ4IO_freePreferences(prefs);
free((void*)inFileNames);
return operationResult;
diff --git a/programs/lz4io.c b/programs/lz4io.c
index a274798..8b70b91 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -1,6 +1,6 @@
/*
LZ4io.c - LZ4 File/Stream Interface
- Copyright (C) Yann Collet 2011-2017
+ Copyright (C) Yann Collet 2011-2020
GPL v2 License
@@ -103,29 +103,9 @@ static int g_displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result
} }
static const clock_t refreshRate = CLOCKS_PER_SEC / 6;
static clock_t g_time = 0;
-#define LZ4IO_STATIC_ASSERT(c) { enum { LZ4IO_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */
-
-/**************************************
-* Local Parameters
-**************************************/
+#define LZ4IO_STATIC_ASSERT(c) { enum { LZ4IO_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */
-struct LZ4IO_prefs_s {
- int passThrough;
- int overwrite;
- int testMode;
- int blockSizeId;
- size_t blockSize;
- int blockChecksum;
- int streamChecksum;
- int blockIndependence;
- int sparseFileSupport;
- int contentSizeFlag;
- int useDictionary;
- unsigned favorDecSpeed;
- const char* dictionaryFilename;
- int removeSrcFile;
-};
/**************************************
* Exceptions
@@ -134,7 +114,7 @@ struct LZ4IO_prefs_s {
# define DEBUG 0
#endif
#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
-#define EXM_THROW(error, ...) \
+#define END_PROCESS(error, ...) \
{ \
DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
DISPLAYLEVEL(1, "Error %i : ", error); \
@@ -144,23 +124,31 @@ struct LZ4IO_prefs_s {
}
-/**************************************
-* Version modifiers
-**************************************/
-#define EXTENDED_ARGUMENTS
-#define EXTENDED_HELP
-#define EXTENDED_FORMAT
-#define DEFAULT_DECOMPRESSOR LZ4IO_decompressLZ4F
-
-
/* ************************************************** */
/* ****************** Parameters ******************** */
/* ************************************************** */
+struct LZ4IO_prefs_s {
+ int passThrough;
+ int overwrite;
+ int testMode;
+ int blockSizeId;
+ size_t blockSize;
+ int blockChecksum;
+ int streamChecksum;
+ int blockIndependence;
+ int sparseFileSupport;
+ int contentSizeFlag;
+ int useDictionary;
+ unsigned favorDecSpeed;
+ const char* dictionaryFilename;
+ int removeSrcFile;
+};
+
LZ4IO_prefs_t* LZ4IO_defaultPreferences(void)
{
LZ4IO_prefs_t* const ret = (LZ4IO_prefs_t*)malloc(sizeof(*ret));
- if (!ret) EXM_THROW(21, "Allocation error : not enough memory");
+ if (!ret) END_PROCESS(21, "Allocation error : not enough memory");
ret->passThrough = 0;
ret->overwrite = 1;
ret->testMode = 0;
@@ -298,6 +286,26 @@ void LZ4IO_setRemoveSrcFile(LZ4IO_prefs_t* const prefs, unsigned flag)
/* ************************************************************************ **
+** ********************** String functions ********************* **
+** ************************************************************************ */
+
+static int LZ4IO_isDevNull(const char* s)
+{
+ return UTIL_sameString(s, nulmark);
+}
+
+static int LZ4IO_isStdin(const char* s)
+{
+ return UTIL_sameString(s, stdinmark);
+}
+
+static int LZ4IO_isStdout(const char* s)
+{
+ return UTIL_sameString(s, stdoutmark);
+}
+
+
+/* ************************************************************************ **
** ********************** LZ4 File / Pipe compression ********************* **
** ************************************************************************ */
@@ -313,13 +321,13 @@ static FILE* LZ4IO_openSrcFile(const char* srcFileName)
{
FILE* f;
- if (!strcmp (srcFileName, stdinmark)) {
- DISPLAYLEVEL(4,"Using stdin for input\n");
+ if (LZ4IO_isStdin(srcFileName)) {
+ DISPLAYLEVEL(4,"Using stdin for input \n");
f = stdin;
SET_BINARY_MODE(stdin);
} else {
f = fopen(srcFileName, "rb");
- if ( f==NULL ) DISPLAYLEVEL(1, "%s: %s \n", srcFileName, strerror(errno));
+ if (f==NULL) DISPLAYLEVEL(1, "%s: %s \n", srcFileName, strerror(errno));
}
return f;
@@ -334,7 +342,7 @@ static FILE* LZ4IO_openDstFile(const char* dstFileName, const LZ4IO_prefs_t* con
FILE* f;
assert(dstFileName != NULL);
- if (!strcmp (dstFileName, stdoutmark)) {
+ if (LZ4IO_isStdout(dstFileName)) {
DISPLAYLEVEL(4, "Using stdout for output \n");
f = stdout;
SET_BINARY_MODE(stdout);
@@ -343,7 +351,8 @@ static FILE* LZ4IO_openDstFile(const char* dstFileName, const LZ4IO_prefs_t* con
" to force-enable it, add --sparse command \n");
}
} else {
- if (!prefs->overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */
+ if (!prefs->overwrite && !LZ4IO_isDevNull(dstFileName)) {
+ /* Check if destination file already exists */
FILE* const testf = fopen( dstFileName, "rb" );
if (testf != NULL) { /* dest exists, prompt for overwrite authorization */
fclose(testf);
@@ -351,7 +360,7 @@ static FILE* LZ4IO_openDstFile(const char* dstFileName, const LZ4IO_prefs_t* con
DISPLAY("%s already exists; not overwritten \n", dstFileName);
return NULL;
}
- DISPLAY("%s already exists; do you wish to overwrite (y/N) ? ", dstFileName);
+ DISPLAY("%s already exists; do you want to overwrite (y/N) ? ", dstFileName);
{ int ch = getchar();
if ((ch!='Y') && (ch!='y')) {
DISPLAY(" not overwritten \n");
@@ -377,7 +386,11 @@ static FILE* LZ4IO_openDstFile(const char* dstFileName, const LZ4IO_prefs_t* con
* Legacy Compression
***************************************/
-/* unoptimized version; solves endianess & alignment issues */
+/* Size in bytes of a legacy block header in little-endian format */
+#define LZ4IO_LEGACY_BLOCK_HEADER_SIZE 4
+#define LZ4IO_LEGACY_BLOCK_SIZE_MAX (8 MB)
+
+/* unoptimized version; solves endianness & alignment issues */
static void LZ4IO_writeLE32 (void* p, unsigned value32)
{
unsigned char* const dstPtr = (unsigned char*)p;
@@ -413,24 +426,24 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
/* Init */
clock_t const clockStart = clock();
if (finput == NULL)
- EXM_THROW(20, "%s : open file error ", input_filename);
+ END_PROCESS(20, "%s : open file error ", input_filename);
foutput = LZ4IO_openDstFile(output_filename, prefs);
if (foutput == NULL) {
fclose(finput);
- EXM_THROW(20, "%s : open file error ", input_filename);
+ END_PROCESS(20, "%s : open file error ", input_filename);
}
/* Allocate Memory */
in_buff = (char*)malloc(LEGACY_BLOCKSIZE);
out_buff = (char*)malloc((size_t)outBuffSize + 4);
if (!in_buff || !out_buff)
- EXM_THROW(21, "Allocation error : not enough memory");
+ END_PROCESS(21, "Allocation error : not enough memory");
/* Write Archive Header */
LZ4IO_writeLE32(out_buff, LEGACY_MAGICNUMBER);
if (fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput) != MAGICNUMBER_SIZE)
- EXM_THROW(22, "Write error : cannot write header");
+ END_PROCESS(22, "Write error : cannot write header");
/* Main Loop */
while (1) {
@@ -445,7 +458,7 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
outSize = compressionFunction(in_buff, out_buff+4, (int)inSize, outBuffSize, compressionlevel);
assert(outSize >= 0);
compressedfilesize += (unsigned long long)outSize+4;
- DISPLAYUPDATE(2, "\rRead : %i MB ==> %.2f%% ",
+ DISPLAYUPDATE(2, "\rRead : %i MiB ==> %.2f%% ",
(int)(filesize>>20), (double)compressedfilesize/filesize*100);
/* Write Block */
@@ -453,19 +466,19 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
assert(outSize < outBuffSize);
LZ4IO_writeLE32(out_buff, (unsigned)outSize);
if (fwrite(out_buff, 1, (size_t)outSize+4, foutput) != (size_t)(outSize+4)) {
- EXM_THROW(24, "Write error : cannot write compressed block");
+ END_PROCESS(24, "Write error : cannot write compressed block");
} }
- if (ferror(finput)) EXM_THROW(25, "Error while reading %s ", input_filename);
+ if (ferror(finput)) END_PROCESS(24, "Error while reading %s ", input_filename);
/* Status */
clockEnd = clock();
- if (clockEnd==clockStart) clockEnd+=1; /* avoid division by zero (speed) */
+ clockEnd += (clockEnd==clockStart); /* avoid division by zero (speed) */
filesize += !filesize; /* avoid division by zero (ratio) */
DISPLAYLEVEL(2, "\r%79s\r", ""); /* blank line */
DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
filesize, compressedfilesize, (double)compressedfilesize / filesize * 100);
{ double const seconds = (double)(clockEnd - clockStart) / CLOCKS_PER_SEC;
- DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds,
+ DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MiB/s\n", seconds,
(double)filesize / seconds / 1024 / 1024);
}
@@ -473,7 +486,7 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
free(in_buff);
free(out_buff);
fclose(finput);
- if (strcmp(output_filename,stdoutmark)) fclose(foutput); /* do not close stdout */
+ if (!LZ4IO_isStdout(output_filename)) fclose(foutput); /* do not close stdout */
return 0;
}
@@ -498,7 +511,7 @@ int LZ4IO_compressMultipleFilenames_Legacy(
/* loop on each file */
for (i=0; i<ifntSize; i++) {
size_t const ifnSize = strlen(inFileNamesTable[i]);
- if (!strcmp(suffix, stdoutmark)) {
+ if (LZ4IO_isStdout(suffix)) {
missed_files += LZ4IO_compressFilename_Legacy(
inFileNamesTable[i], stdoutmark,
compressionLevel, prefs);
@@ -530,7 +543,6 @@ int LZ4IO_compressMultipleFilenames_Legacy(
/*********************************************
* Compression using Frame format
*********************************************/
-
typedef struct {
void* srcBuffer;
size_t srcBufferSize;
@@ -551,15 +563,15 @@ static void* LZ4IO_createDict(size_t* dictSize, const char* const dictFilename)
char* dictBuf;
FILE* dictFile;
- if (!circularBuf) EXM_THROW(25, "Allocation error : not enough memory for circular buffer");
- if (!dictFilename) EXM_THROW(25, "Dictionary error : no filename provided");
+ if (!circularBuf) END_PROCESS(25, "Allocation error : not enough memory for circular buffer");
+ if (!dictFilename) END_PROCESS(26, "Dictionary error : no filename provided");
dictFile = LZ4IO_openSrcFile(dictFilename);
- if (!dictFile) EXM_THROW(25, "Dictionary error : could not open dictionary file");
+ if (!dictFile) END_PROCESS(27, "Dictionary error : could not open dictionary file");
- /* opportunistically seek to the part of the file we care about. If this */
- /* fails it's not a problem since we'll just read everything anyways. */
- if (strcmp(dictFilename, stdinmark)) {
+ /* opportunistically seek to the part of the file we care about.
+ * If this fails it's not a problem since we'll just read everything anyways. */
+ if (!LZ4IO_isStdin(dictFilename)) {
(void)UTIL_fseek(dictFile, -LZ4_MAX_DICT_SIZE, SEEK_END);
}
@@ -584,7 +596,7 @@ static void* LZ4IO_createDict(size_t* dictSize, const char* const dictFilename)
} else {
/* Otherwise, we will alloc a new buffer and copy our dict into that. */
dictBuf = (char *)malloc(dictLen ? dictLen : 1);
- if (!dictBuf) EXM_THROW(25, "Allocation error : not enough memory");
+ if (!dictBuf) END_PROCESS(28, "Allocation error : not enough memory");
memcpy(dictBuf, circularBuf + dictStart, circularBufSize - dictStart);
memcpy(dictBuf + circularBufSize - dictStart, circularBuf, dictLen - (circularBufSize - dictStart));
@@ -603,7 +615,7 @@ static LZ4F_CDict* LZ4IO_createCDict(const LZ4IO_prefs_t* const prefs)
LZ4F_CDict* cdict;
if (!prefs->useDictionary) return NULL;
dictionaryBuffer = LZ4IO_createDict(&dictionarySize, prefs->dictionaryFilename);
- if (!dictionaryBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary");
+ if (!dictionaryBuffer) END_PROCESS(29, "Dictionary error : could not create dictionary");
cdict = LZ4F_createCDict(dictionaryBuffer, dictionarySize);
free(dictionaryBuffer);
return cdict;
@@ -615,14 +627,14 @@ static cRess_t LZ4IO_createCResources(const LZ4IO_prefs_t* const prefs)
cRess_t ress;
LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&(ress.ctx), LZ4F_VERSION);
- if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode));
+ if (LZ4F_isError(errorCode)) END_PROCESS(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode));
/* Allocate Memory */
ress.srcBuffer = malloc(blockSize);
ress.srcBufferSize = blockSize;
ress.dstBufferSize = LZ4F_compressFrameBound(blockSize, NULL); /* cover worst case */
ress.dstBuffer = malloc(ress.dstBufferSize);
- if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory");
+ if (!ress.srcBuffer || !ress.dstBuffer) END_PROCESS(31, "Allocation error : not enough memory");
ress.cdict = LZ4IO_createCDict(prefs);
@@ -638,7 +650,7 @@ static void LZ4IO_freeCResources(cRess_t ress)
ress.cdict = NULL;
{ LZ4F_errorCode_t const errorCode = LZ4F_freeCompressionContext(ress.ctx);
- if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); }
+ if (LZ4F_isError(errorCode)) END_PROCESS(35, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); }
}
/*
@@ -686,7 +698,7 @@ LZ4IO_compressFilename_extRess(cRess_t ress,
/* read first block */
readSize = fread(srcBuffer, (size_t)1, blockSize, srcFile);
- if (ferror(srcFile)) EXM_THROW(30, "Error reading %s ", srcFileName);
+ if (ferror(srcFile)) END_PROCESS(40, "Error reading %s ", srcFileName);
filesize += readSize;
/* single-block file */
@@ -694,14 +706,14 @@ LZ4IO_compressFilename_extRess(cRess_t ress,
/* Compress in single pass */
size_t const cSize = LZ4F_compressFrame_usingCDict(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs);
if (LZ4F_isError(cSize))
- EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize));
+ END_PROCESS(41, "Compression failed : %s", LZ4F_getErrorName(cSize));
compressedfilesize = cSize;
- DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
+ DISPLAYUPDATE(2, "\rRead : %u MiB ==> %.2f%% ",
(unsigned)(filesize>>20), (double)compressedfilesize/(filesize+!filesize)*100); /* avoid division by zero */
/* Write Block */
if (fwrite(dstBuffer, 1, cSize, dstFile) != cSize) {
- EXM_THROW(32, "Write error : failed writing single-block compressed frame");
+ END_PROCESS(42, "Write error : failed writing single-block compressed frame");
} }
else
@@ -710,55 +722,55 @@ LZ4IO_compressFilename_extRess(cRess_t ress,
{
/* Write Frame Header */
size_t const headerSize = LZ4F_compressBegin_usingCDict(ctx, dstBuffer, dstBufferSize, ress.cdict, &prefs);
- if (LZ4F_isError(headerSize)) EXM_THROW(33, "File header generation failed : %s", LZ4F_getErrorName(headerSize));
+ if (LZ4F_isError(headerSize)) END_PROCESS(43, "File header generation failed : %s", LZ4F_getErrorName(headerSize));
if (fwrite(dstBuffer, 1, headerSize, dstFile) != headerSize)
- EXM_THROW(34, "Write error : cannot write header");
+ END_PROCESS(44, "Write error : cannot write header");
compressedfilesize += headerSize;
/* Main Loop - one block at a time */
while (readSize>0) {
size_t const outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL);
if (LZ4F_isError(outSize))
- EXM_THROW(35, "Compression failed : %s", LZ4F_getErrorName(outSize));
+ END_PROCESS(45, "Compression failed : %s", LZ4F_getErrorName(outSize));
compressedfilesize += outSize;
- DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
+ DISPLAYUPDATE(2, "\rRead : %u MiB ==> %.2f%% ",
(unsigned)(filesize>>20), (double)compressedfilesize/filesize*100);
/* Write Block */
if (fwrite(dstBuffer, 1, outSize, dstFile) != outSize)
- EXM_THROW(36, "Write error : cannot write compressed block");
+ END_PROCESS(46, "Write error : cannot write compressed block");
/* Read next block */
readSize = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile);
filesize += readSize;
}
- if (ferror(srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName);
+ if (ferror(srcFile)) END_PROCESS(47, "Error reading %s ", srcFileName);
/* End of Frame mark */
{ size_t const endSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL);
if (LZ4F_isError(endSize))
- EXM_THROW(38, "End of frame error : %s", LZ4F_getErrorName(endSize));
+ END_PROCESS(48, "End of frame error : %s", LZ4F_getErrorName(endSize));
if (fwrite(dstBuffer, 1, endSize, dstFile) != endSize)
- EXM_THROW(39, "Write error : cannot write end of frame");
+ END_PROCESS(49, "Write error : cannot write end of frame");
compressedfilesize += endSize;
} }
/* Release file handlers */
fclose (srcFile);
- if (strcmp(dstFileName,stdoutmark)) fclose (dstFile); /* do not close stdout */
+ if (!LZ4IO_isStdout(dstFileName)) fclose(dstFile); /* do not close stdout */
/* Copy owner, file permissions and modification time */
{ stat_t statbuf;
- if (strcmp (srcFileName, stdinmark)
- && strcmp (dstFileName, stdoutmark)
- && strcmp (dstFileName, nulmark)
+ if (!LZ4IO_isStdin(srcFileName)
+ && !LZ4IO_isStdout(dstFileName)
+ && !LZ4IO_isDevNull(dstFileName)
&& UTIL_getFileStat(srcFileName, &statbuf)) {
UTIL_setFileStat(dstFileName, &statbuf);
} }
if (io_prefs->removeSrcFile) { /* remove source file : --rm */
if (remove(srcFileName))
- EXM_THROW(40, "Remove error : %s: %s", srcFileName, strerror(errno));
+ END_PROCESS(50, "Remove error : %s: %s", srcFileName, strerror(errno));
}
/* Final Status */
@@ -814,12 +826,13 @@ int LZ4IO_compressMultipleFilenames(
/* loop on each file */
for (i=0; i<ifntSize; i++) {
size_t const ifnSize = strlen(inFileNamesTable[i]);
- if (!strcmp(suffix, stdoutmark)) {
+ if (LZ4IO_isStdout(suffix)) {
missed_files += LZ4IO_compressFilename_extRess(ress,
inFileNamesTable[i], stdoutmark,
compressionLevel, prefs);
continue;
}
+ /* suffix != stdout => compress into a file => generate its name */
if (ofnSize <= ifnSize+suffixSize+1) {
free(dstFileName);
ofnSize = ifnSize + 20;
@@ -877,14 +890,14 @@ LZ4IO_fwriteSparse(FILE* file,
if (!sparseMode) { /* normal write */
size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file);
- if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : cannot write decoded block");
+ if (sizeCheck != bufferSize) END_PROCESS(70, "Write error : cannot write decoded block");
return 0;
}
/* avoid int overflow */
if (storedSkips > 1 GB) {
int const seekResult = UTIL_fseek(file, 1 GB, SEEK_CUR);
- if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)");
+ if (seekResult != 0) END_PROCESS(71, "1 GB skip error (sparse file support)");
storedSkips -= 1 GB;
}
@@ -901,13 +914,13 @@ LZ4IO_fwriteSparse(FILE* file,
if (nb0T != seg0SizeT) { /* not all 0s */
errno = 0;
{ int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR);
- if (seekResult) EXM_THROW(72, "Sparse skip error(%d): %s ; try --no-sparse", (int)errno, strerror(errno));
+ if (seekResult) END_PROCESS(72, "Sparse skip error(%d): %s ; try --no-sparse", (int)errno, strerror(errno));
}
storedSkips = 0;
seg0SizeT -= nb0T;
ptrT += nb0T;
{ size_t const sizeCheck = fwrite(ptrT, sizeT, seg0SizeT, file);
- if (sizeCheck != seg0SizeT) EXM_THROW(73, "Write error : cannot write decoded block");
+ if (sizeCheck != seg0SizeT) END_PROCESS(73, "Write error : cannot write decoded block");
} }
ptrT += seg0SizeT;
}
@@ -921,10 +934,10 @@ LZ4IO_fwriteSparse(FILE* file,
storedSkips += (unsigned) (restPtr - restStart);
if (restPtr != restEnd) {
int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR);
- if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse");
+ if (seekResult) END_PROCESS(74, "Sparse skip error ; try --no-sparse");
storedSkips = 0;
{ size_t const sizeCheck = fwrite(restPtr, 1, (size_t)(restEnd - restPtr), file);
- if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(75, "Write error : cannot write decoded end of block");
+ if (sizeCheck != (size_t)(restEnd - restPtr)) END_PROCESS(75, "Write error : cannot write decoded end of block");
} }
}
@@ -936,15 +949,17 @@ static void LZ4IO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
if (storedSkips>0) { /* implies sparseFileSupport>0 */
const char lastZeroByte[1] = { 0 };
if (UTIL_fseek(file, storedSkips-1, SEEK_CUR) != 0)
- EXM_THROW(69, "Final skip error (sparse file)\n");
+ END_PROCESS(68, "Final skip error (sparse file)\n");
if (fwrite(lastZeroByte, 1, 1, file) != 1)
- EXM_THROW(69, "Write error : cannot write last zero\n");
+ END_PROCESS(69, "Write error : cannot write last zero\n");
}
}
static unsigned g_magicRead = 0; /* out-parameter of LZ4IO_decodeLegacyStream() */
-static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput, const LZ4IO_prefs_t* prefs)
+
+static unsigned long long
+LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput, const LZ4IO_prefs_t* prefs)
{
unsigned long long streamSize = 0;
unsigned storedSkips = 0;
@@ -952,18 +967,19 @@ static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput,
/* Allocate Memory */
char* const in_buff = (char*)malloc((size_t)LZ4_compressBound(LEGACY_BLOCKSIZE));
char* const out_buff = (char*)malloc(LEGACY_BLOCKSIZE);
- if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory");
+ if (!in_buff || !out_buff) END_PROCESS(51, "Allocation error : not enough memory");
/* Main Loop */
while (1) {
unsigned int blockSize;
/* Block Size */
- { size_t const sizeCheck = fread(in_buff, 1, 4, finput);
+ { size_t const sizeCheck = fread(in_buff, 1, LZ4IO_LEGACY_BLOCK_HEADER_SIZE, finput);
if (sizeCheck == 0) break; /* Nothing to read : file read is completed */
- if (sizeCheck != 4) EXM_THROW(52, "Read error : cannot access block size "); }
- blockSize = LZ4IO_readLE32(in_buff); /* Convert to Little Endian */
- if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) {
+ if (sizeCheck != LZ4IO_LEGACY_BLOCK_HEADER_SIZE) END_PROCESS(52, "Read error : cannot access block size ");
+ }
+ blockSize = LZ4IO_readLE32(in_buff); /* Convert to Little Endian */
+ if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) {
/* Cannot read next block : maybe new stream ? */
g_magicRead = blockSize;
break;
@@ -971,16 +987,16 @@ static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput,
/* Read Block */
{ size_t const sizeCheck = fread(in_buff, 1, blockSize, finput);
- if (sizeCheck!=blockSize) EXM_THROW(52, "Read error : cannot access compressed block !"); }
+ if (sizeCheck != blockSize) END_PROCESS(53, "Read error : cannot access compressed block !"); }
/* Decode Block */
{ int const decodeSize = LZ4_decompress_safe(in_buff, out_buff, (int)blockSize, LEGACY_BLOCKSIZE);
- if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !");
+ if (decodeSize < 0) END_PROCESS(54, "Decoding Failed ! Corrupted input detected !");
streamSize += (unsigned long long)decodeSize;
/* Write Block */
storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, (size_t)decodeSize, prefs->sparseFileSupport, storedSkips); /* success or die */
} }
- if (ferror(finput)) EXM_THROW(54, "Read error : ferror");
+ if (ferror(finput)) END_PROCESS(55, "Read error : ferror");
LZ4IO_fwriteSparseEnd(foutput, storedSkips);
@@ -1013,7 +1029,7 @@ static void LZ4IO_loadDDict(dRess_t* ress, const LZ4IO_prefs_t* const prefs)
}
ress->dictBuffer = LZ4IO_createDict(&ress->dictBufferSize, prefs->dictionaryFilename);
- if (!ress->dictBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary");
+ if (!ress->dictBuffer) END_PROCESS(25, "Dictionary error : could not create dictionary");
}
static const size_t LZ4IO_dBufferSize = 64 KB;
@@ -1023,14 +1039,14 @@ static dRess_t LZ4IO_createDResources(const LZ4IO_prefs_t* const prefs)
/* init */
LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&ress.dCtx, LZ4F_VERSION);
- if (LZ4F_isError(errorCode)) EXM_THROW(60, "Can't create LZ4F context : %s", LZ4F_getErrorName(errorCode));
+ if (LZ4F_isError(errorCode)) END_PROCESS(60, "Can't create LZ4F context : %s", LZ4F_getErrorName(errorCode));
/* Allocate Memory */
ress.srcBufferSize = LZ4IO_dBufferSize;
ress.srcBuffer = malloc(ress.srcBufferSize);
ress.dstBufferSize = LZ4IO_dBufferSize;
ress.dstBuffer = malloc(ress.dstBufferSize);
- if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
+ if (!ress.srcBuffer || !ress.dstBuffer) END_PROCESS(61, "Allocation error : not enough memory");
LZ4IO_loadDDict(&ress, prefs);
@@ -1041,7 +1057,7 @@ static dRess_t LZ4IO_createDResources(const LZ4IO_prefs_t* const prefs)
static void LZ4IO_freeDResources(dRess_t ress)
{
LZ4F_errorCode_t errorCode = LZ4F_freeDecompressionContext(ress.dCtx);
- if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode));
+ if (LZ4F_isError(errorCode)) END_PROCESS(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode));
free(ress.srcBuffer);
free(ress.dstBuffer);
free(ress.dictBuffer);
@@ -1056,13 +1072,22 @@ LZ4IO_decompressLZ4F(dRess_t ress,
unsigned long long filesize = 0;
LZ4F_errorCode_t nextToLoad;
unsigned storedSkips = 0;
+ LZ4F_decompressOptions_t const dOpt_skipCrc = { 0, 1, 0, 0 };
+ const LZ4F_decompressOptions_t* const dOptPtr =
+ ((prefs->blockChecksum==0) && (prefs->streamChecksum==0)) ?
+ &dOpt_skipCrc : NULL;
/* Init feed with magic number (already consumed from FILE* sFile) */
{ size_t inSize = MAGICNUMBER_SIZE;
size_t outSize= 0;
LZ4IO_writeLE32(ress.srcBuffer, LZ4IO_MAGICNUMBER);
- nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, ress.dstBuffer, &outSize, ress.srcBuffer, &inSize, ress.dictBuffer, ress.dictBufferSize, NULL);
- if (LZ4F_isError(nextToLoad)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(nextToLoad));
+ nextToLoad = LZ4F_decompress_usingDict(ress.dCtx,
+ ress.dstBuffer, &outSize,
+ ress.srcBuffer, &inSize,
+ ress.dictBuffer, ress.dictBufferSize,
+ dOptPtr); /* set it once, it's enough */
+ if (LZ4F_isError(nextToLoad))
+ END_PROCESS(62, "Header error : %s", LZ4F_getErrorName(nextToLoad));
}
/* Main Loop */
@@ -1080,8 +1105,13 @@ LZ4IO_decompressLZ4F(dRess_t ress,
/* Decode Input (at least partially) */
size_t remaining = readSize - pos;
decodedBytes = ress.dstBufferSize;
- nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, ress.dictBuffer, ress.dictBufferSize, NULL);
- if (LZ4F_isError(nextToLoad)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad));
+ nextToLoad = LZ4F_decompress_usingDict(ress.dCtx,
+ ress.dstBuffer, &decodedBytes,
+ (char*)(ress.srcBuffer)+pos, &remaining,
+ ress.dictBuffer, ress.dictBufferSize,
+ NULL);
+ if (LZ4F_isError(nextToLoad))
+ END_PROCESS(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad));
pos += remaining;
/* Write Block */
@@ -1089,17 +1119,17 @@ LZ4IO_decompressLZ4F(dRess_t ress,
if (!prefs->testMode)
storedSkips = LZ4IO_fwriteSparse(dstFile, ress.dstBuffer, decodedBytes, prefs->sparseFileSupport, storedSkips);
filesize += decodedBytes;
- DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20));
+ DISPLAYUPDATE(2, "\rDecompressed : %u MiB ", (unsigned)(filesize>>20));
}
if (!nextToLoad) break;
}
}
/* can be out because readSize == 0, which could be an fread() error */
- if (ferror(srcFile)) EXM_THROW(67, "Read error");
+ if (ferror(srcFile)) END_PROCESS(67, "Read error");
if (!prefs->testMode) LZ4IO_fwriteSparseEnd(dstFile, storedSkips);
- if (nextToLoad!=0) EXM_THROW(68, "Unfinished stream");
+ if (nextToLoad!=0) END_PROCESS(68, "Unfinished stream");
return filesize;
}
@@ -1123,19 +1153,36 @@ LZ4IO_passThrough(FILE* finput, FILE* foutput,
unsigned storedSkips = 0;
if (fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput) != MAGICNUMBER_SIZE) {
- EXM_THROW(50, "Pass-through write error");
+ END_PROCESS(50, "Pass-through write error");
}
while (readBytes) {
readBytes = fread(buffer, 1, sizeof(buffer), finput);
total += readBytes;
storedSkips = LZ4IO_fwriteSparse(foutput, buffer, readBytes, sparseFileSupport, storedSkips);
}
- if (ferror(finput)) EXM_THROW(51, "Read Error");
+ if (ferror(finput)) END_PROCESS(51, "Read Error");
LZ4IO_fwriteSparseEnd(foutput, storedSkips);
return total;
}
+/* when fseek() doesn't work (pipe scenario),
+ * read and forget from input.
+**/
+#define SKIP_BUFF_SIZE (16 KB)
+#define MIN(a,b) ( ((a)<(b)) ? (a) : (b) )
+static int skipStream(FILE* f, unsigned offset)
+{
+ char buf[SKIP_BUFF_SIZE];
+ while (offset > 0) {
+ size_t const tr = MIN(offset, sizeof(buf));
+ size_t const r = fread(buf, 1, tr, f);
+ if (r != tr) return 1; /* error reading f */
+ offset -= (unsigned)tr;
+ }
+ assert(offset == 0);
+ return 0;
+}
/** Safely handle cases when (unsigned)offset > LONG_MAX */
static int fseek_u32(FILE *fp, unsigned offset, int where)
@@ -1147,14 +1194,17 @@ static int fseek_u32(FILE *fp, unsigned offset, int where)
while (offset > 0) {
unsigned s = offset;
if (s > stepMax) s = stepMax;
- errorNb = UTIL_fseek(fp, (long) s, SEEK_CUR);
- if (errorNb != 0) break;
- offset -= s;
+ errorNb = UTIL_fseek(fp, (long)s, SEEK_CUR);
+ if (errorNb==0) { offset -= s; continue; }
+ errorNb = skipStream(fp, offset);
+ offset = 0;
}
return errorNb;
}
+
#define ENDOFSTREAM ((unsigned long long)-1)
+#define DECODING_ERROR ((unsigned long long)-2)
static unsigned long long
selectDecoder(dRess_t ress,
FILE* finput, FILE* foutput,
@@ -1175,7 +1225,7 @@ selectDecoder(dRess_t ress,
size_t const nbReadBytes = fread(MNstore, 1, MAGICNUMBER_SIZE, finput);
if (nbReadBytes==0) { nbFrames = 0; return ENDOFSTREAM; } /* EOF */
if (nbReadBytes != MAGICNUMBER_SIZE)
- EXM_THROW(40, "Unrecognized header : Magic Number unreadable");
+ END_PROCESS(40, "Unrecognized header : Magic Number unreadable");
magicNumber = LZ4IO_readLE32(MNstore); /* Little Endian format */
}
if (LZ4IO_isSkippableMagicNumber(magicNumber))
@@ -1192,15 +1242,14 @@ selectDecoder(dRess_t ress,
DISPLAYLEVEL(4, "Skipping detected skippable area \n");
{ size_t const nbReadBytes = fread(MNstore, 1, 4, finput);
if (nbReadBytes != 4)
- EXM_THROW(42, "Stream error : skippable size unreadable");
+ END_PROCESS(42, "Stream error : skippable size unreadable");
}
{ unsigned const size = LZ4IO_readLE32(MNstore);
int const errorNb = fseek_u32(finput, size, SEEK_CUR);
if (errorNb != 0)
- EXM_THROW(43, "Stream error : cannot skip skippable area");
+ END_PROCESS(43, "Stream error : cannot skip skippable area");
}
return 0;
- EXTENDED_FORMAT; /* macro extension for custom formats */
default:
if (nbFrames == 1) { /* just started */
/* Wrong magic number at the beginning of 1st stream */
@@ -1208,7 +1257,7 @@ selectDecoder(dRess_t ress,
nbFrames = 0;
return LZ4IO_passThrough(finput, foutput, MNstore, prefs->sparseFileSupport);
}
- EXM_THROW(44,"Unrecognized header : file cannot be decoded");
+ END_PROCESS(44,"Unrecognized header : file cannot be decoded");
}
{ long int const position = ftell(finput); /* only works for files < 2 GB */
DISPLAYLEVEL(2, "Stream followed by undecodable data ");
@@ -1216,7 +1265,7 @@ selectDecoder(dRess_t ress,
DISPLAYLEVEL(2, "at position %i ", (int)position);
DISPLAYLEVEL(2, "\n");
}
- return ENDOFSTREAM;
+ return DECODING_ERROR;
}
}
@@ -1228,6 +1277,7 @@ LZ4IO_decompressSrcFile(dRess_t ress,
{
FILE* const foutput = ress.dstFile;
unsigned long long filesize = 0;
+ int result = 0;
/* Init */
FILE* const finput = LZ4IO_openSrcFile(input_filename);
@@ -1239,6 +1289,7 @@ LZ4IO_decompressSrcFile(dRess_t ress,
unsigned long long const decodedSize =
selectDecoder(ress, finput, foutput, prefs);
if (decodedSize == ENDOFSTREAM) break;
+ if (decodedSize == DECODING_ERROR) { result=1; break; }
filesize += decodedSize;
}
@@ -1246,7 +1297,7 @@ LZ4IO_decompressSrcFile(dRess_t ress,
fclose(finput);
if (prefs->removeSrcFile) { /* --rm */
if (remove(input_filename))
- EXM_THROW(45, "Remove error : %s: %s", input_filename, strerror(errno));
+ END_PROCESS(45, "Remove error : %s: %s", input_filename, strerror(errno));
}
/* Final Status */
@@ -1254,7 +1305,7 @@ LZ4IO_decompressSrcFile(dRess_t ress,
DISPLAYLEVEL(2, "%-20.20s : decoded %llu bytes \n", input_filename, filesize);
(void)output_filename;
- return 0;
+ return result;
}
@@ -1263,45 +1314,50 @@ LZ4IO_decompressDstFile(dRess_t ress,
const char* input_filename, const char* output_filename,
const LZ4IO_prefs_t* const prefs)
{
+ int result;
stat_t statbuf;
int stat_result = 0;
FILE* const foutput = LZ4IO_openDstFile(output_filename, prefs);
if (foutput==NULL) return 1; /* failure */
- if ( strcmp(input_filename, stdinmark)
+ if ( !LZ4IO_isStdin(input_filename)
&& UTIL_getFileStat(input_filename, &statbuf))
stat_result = 1;
ress.dstFile = foutput;
- LZ4IO_decompressSrcFile(ress, input_filename, output_filename, prefs);
+ result = LZ4IO_decompressSrcFile(ress, input_filename, output_filename, prefs);
fclose(foutput);
/* Copy owner, file permissions and modification time */
if ( stat_result != 0
- && strcmp (output_filename, stdoutmark)
- && strcmp (output_filename, nulmark)) {
+ && !LZ4IO_isStdout(output_filename)
+ && !LZ4IO_isDevNull(output_filename)) {
UTIL_setFileStat(output_filename, &statbuf);
/* should return value be read ? or is silent fail good enough ? */
}
- return 0;
+ return result;
}
+/* Note : LZ4IO_decompressFilename()
+ * can provide total decompression time for the specified fileName.
+ * This information is not available with LZ4IO_decompressMultipleFilenames().
+ */
int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename, const LZ4IO_prefs_t* prefs)
{
dRess_t const ress = LZ4IO_createDResources(prefs);
clock_t const start = clock();
- int const missingFiles = LZ4IO_decompressDstFile(ress, input_filename, output_filename, prefs);
+ int const status = LZ4IO_decompressDstFile(ress, input_filename, output_filename, prefs);
clock_t const end = clock();
double const seconds = (double)(end - start) / CLOCKS_PER_SEC;
DISPLAYLEVEL(4, "Done in %.2f sec \n", seconds);
LZ4IO_freeDResources(ress);
- return missingFiles;
+ return status;
}
@@ -1318,23 +1374,26 @@ int LZ4IO_decompressMultipleFilenames(
size_t const suffixSize = strlen(suffix);
dRess_t ress = LZ4IO_createDResources(prefs);
- if (outFileName==NULL) EXM_THROW(70, "Memory allocation error");
+ if (outFileName==NULL) END_PROCESS(70, "Memory allocation error");
+ if (prefs->blockChecksum==0 && prefs->streamChecksum==0) {
+ DISPLAYLEVEL(4, "disabling checksum validation during decoding \n");
+ }
ress.dstFile = LZ4IO_openDstFile(stdoutmark, prefs);
for (i=0; i<ifntSize; i++) {
size_t const ifnSize = strlen(inFileNamesTable[i]);
const char* const suffixPtr = inFileNamesTable[i] + ifnSize - suffixSize;
- if (!strcmp(suffix, stdoutmark)) {
- missingFiles += LZ4IO_decompressSrcFile(ress, inFileNamesTable[i], stdoutmark, prefs);
+ if (LZ4IO_isStdout(suffix) || LZ4IO_isDevNull(suffix)) {
+ missingFiles += LZ4IO_decompressSrcFile(ress, inFileNamesTable[i], suffix, prefs);
continue;
}
if (ofnSize <= ifnSize-suffixSize+1) {
free(outFileName);
ofnSize = ifnSize + 20;
outFileName = (char*)malloc(ofnSize);
- if (outFileName==NULL) EXM_THROW(71, "Memory allocation error");
+ if (outFileName==NULL) END_PROCESS(71, "Memory allocation error");
}
- if (ifnSize <= suffixSize || strcmp(suffixPtr, suffix) != 0) {
+ if (ifnSize <= suffixSize || !UTIL_sameString(suffixPtr, suffix) ) {
DISPLAYLEVEL(1, "File extension doesn't match expected LZ4_EXTENSION (%4s); will not process file: %s\n", suffix, inFileNamesTable[i]);
skippedFiles++;
continue;
@@ -1387,7 +1446,7 @@ static const char * LZ4IO_frameTypeNames[] = {"LZ4Frame", "LegacyFrame", "Skippa
/* Read block headers and skip block data
Return total blocks size for this frame including block headers,
block checksums and content checksums.
- returns 0 in case it can't succesfully skip block data.
+ returns 0 in case it can't successfully skip block data.
Assumes SEEK_CUR after frame header.
*/
static unsigned long long
@@ -1424,37 +1483,47 @@ LZ4IO_skipBlocksData(FILE* finput,
return totalBlocksSize;
}
+static const unsigned long long legacyFrameUndecodable = (0ULL-1);
/* For legacy frames only.
Read block headers and skip block data.
Return total blocks size for this frame including block headers.
- or 0 in case it can't succesfully skip block data.
+ or legacyFrameUndecodable in case it can't successfully skip block data.
This works as long as legacy block header size = magic number size.
Assumes SEEK_CUR after frame header.
*/
static unsigned long long LZ4IO_skipLegacyBlocksData(FILE* finput)
{
- unsigned char blockInfo[LZIO_LEGACY_BLOCK_HEADER_SIZE];
+ unsigned char blockInfo[LZ4IO_LEGACY_BLOCK_HEADER_SIZE];
unsigned long long totalBlocksSize = 0;
- LZ4IO_STATIC_ASSERT(LZIO_LEGACY_BLOCK_HEADER_SIZE == MAGICNUMBER_SIZE);
+ LZ4IO_STATIC_ASSERT(LZ4IO_LEGACY_BLOCK_HEADER_SIZE == MAGICNUMBER_SIZE);
for (;;) {
- if (!fread(blockInfo, 1, LZIO_LEGACY_BLOCK_HEADER_SIZE, finput)) {
+ size_t const bhs = fread(blockInfo, 1, LZ4IO_LEGACY_BLOCK_HEADER_SIZE, finput);
+ if (bhs == 0) {
if (feof(finput)) return totalBlocksSize;
- return 0;
+ return legacyFrameUndecodable;
+ }
+ if (bhs != 4) {
+ return legacyFrameUndecodable;
}
{ const unsigned int nextCBlockSize = LZ4IO_readLE32(&blockInfo);
- if ( nextCBlockSize == LEGACY_MAGICNUMBER ||
- nextCBlockSize == LZ4IO_MAGICNUMBER ||
- LZ4IO_isSkippableMagicNumber(nextCBlockSize)) {
- /* Rewind back. we want cursor at the begining of next frame.*/
- if (fseek(finput, -LZIO_LEGACY_BLOCK_HEADER_SIZE, SEEK_CUR) != 0) {
- return 0;
+ if ( nextCBlockSize == LEGACY_MAGICNUMBER
+ || nextCBlockSize == LZ4IO_MAGICNUMBER
+ || LZ4IO_isSkippableMagicNumber(nextCBlockSize) ) {
+ /* Rewind back. we want cursor at the beginning of next frame */
+ if (UTIL_fseek(finput, -LZ4IO_LEGACY_BLOCK_HEADER_SIZE, SEEK_CUR) != 0) {
+ END_PROCESS(37, "impossible to skip backward");
}
break;
}
- totalBlocksSize += LZIO_LEGACY_BLOCK_HEADER_SIZE + nextCBlockSize;
- /* skip to the next block */
+ if (nextCBlockSize > LZ4IO_LEGACY_BLOCK_SIZE_MAX) {
+ DISPLAYLEVEL(4, "Error : block in legacy frame is too large \n");
+ return legacyFrameUndecodable;
+ }
+ totalBlocksSize += LZ4IO_LEGACY_BLOCK_HEADER_SIZE + nextCBlockSize;
+ /* skip to the next block
+ * note : this won't fail if nextCBlockSize is too large, skipping past the end of finput */
if (UTIL_fseek(finput, nextCBlockSize, SEEK_CUR) != 0) {
- return 0;
+ return legacyFrameUndecodable;
} } }
return totalBlocksSize;
}
@@ -1514,7 +1583,7 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam
if (nbReadBytes == 0) { break; } /* EOF */
result = LZ4IO_format_not_known; /* default result (error) */
if (nbReadBytes != MAGICNUMBER_SIZE) {
- EXM_THROW(40, "Unrecognized header : Magic Number unreadable");
+ END_PROCESS(40, "Unrecognized header : Magic Number unreadable");
} }
magicNumber = LZ4IO_readLE32(buffer); /* Little Endian format */
if (LZ4IO_isSkippableMagicNumber(magicNumber))
@@ -1525,14 +1594,14 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam
if (cfinfo->frameSummary.frameType != lz4Frame) cfinfo->eqFrameTypes = 0;
/* Get frame info */
{ const size_t readBytes = fread(buffer + MAGICNUMBER_SIZE, 1, LZ4F_HEADER_SIZE_MIN - MAGICNUMBER_SIZE, finput);
- if (!readBytes || ferror(finput)) EXM_THROW(71, "Error reading %s", input_filename);
+ if (!readBytes || ferror(finput)) END_PROCESS(71, "Error reading %s", input_filename);
}
{ size_t hSize = LZ4F_headerSize(&buffer, LZ4F_HEADER_SIZE_MIN);
if (LZ4F_isError(hSize)) break;
if (hSize > (LZ4F_HEADER_SIZE_MIN + MAGICNUMBER_SIZE)) {
/* We've already read LZ4F_HEADER_SIZE_MIN so read any extra until hSize*/
const size_t readBytes = fread(buffer + LZ4F_HEADER_SIZE_MIN, 1, hSize - LZ4F_HEADER_SIZE_MIN, finput);
- if (!readBytes || ferror(finput)) EXM_THROW(72, "Error reading %s", input_filename);
+ if (!readBytes || ferror(finput)) END_PROCESS(72, "Error reading %s", input_filename);
}
/* Create decompression context */
{ LZ4F_dctx* dctx;
@@ -1578,6 +1647,11 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam
cfinfo->eqBlockTypes = 0;
cfinfo->allContentSize = 0;
{ const unsigned long long totalBlocksSize = LZ4IO_skipLegacyBlocksData(finput);
+ if (totalBlocksSize == legacyFrameUndecodable) {
+ DISPLAYLEVEL(1, "Corrupted legacy frame \n");
+ result = LZ4IO_format_not_known;
+ break;
+ }
if (totalBlocksSize) {
DISPLAYLEVEL(3, " %6llu %14s %5s %8s %20llu %20s %9s\n",
cfinfo->frameCount + 1,
@@ -1595,12 +1669,12 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam
cfinfo->allContentSize = 0;
{ size_t const nbReadBytes = fread(buffer, 1, 4, finput);
if (nbReadBytes != 4)
- EXM_THROW(42, "Stream error : skippable size unreadable");
+ END_PROCESS(42, "Stream error : skippable size unreadable");
}
{ unsigned const size = LZ4IO_readLE32(buffer);
int const errorNb = fseek_u32(finput, size, SEEK_CUR);
if (errorNb != 0)
- EXM_THROW(43, "Stream error : cannot skip skippable area");
+ END_PROCESS(43, "Stream error : cannot skip skippable area");
DISPLAYLEVEL(3, " %6llu %14s %5s %8s %20u %20s %9s\n",
cfinfo->frameCount + 1,
"SkippableFrame",
@@ -1614,6 +1688,7 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam
DISPLAYLEVEL(3, "Stream followed by undecodable data ");
if (position != -1L)
DISPLAYLEVEL(3, "at position %i ", (int)position);
+ result = LZ4IO_format_not_known;
DISPLAYLEVEL(3, "\n");
}
break;
@@ -1639,9 +1714,9 @@ int LZ4IO_displayCompressedFilesInfo(const char** inFileNames, size_t ifnIdx)
/* Get file info */
LZ4IO_cFileInfo_t cfinfo = LZ4IO_INIT_CFILEINFO;
cfinfo.fileName = LZ4IO_baseName(inFileNames[idx]);
- if (!UTIL_isRegFile(inFileNames[idx])) {
+ if (LZ4IO_isStdin(inFileNames[idx]) ? !UTIL_isRegFD(0) : !UTIL_isRegFile(inFileNames[idx])) {
DISPLAYLEVEL(1, "lz4: %s is not a regular file \n", inFileNames[idx]);
- return 0;
+ return 1;
}
DISPLAYLEVEL(3, "%s(%llu/%llu)\n", cfinfo.fileName, (unsigned long long)idx + 1, (unsigned long long)ifnIdx);
DISPLAYLEVEL(3, " %6s %14s %5s %8s %20s %20s %9s\n",
@@ -1650,7 +1725,7 @@ int LZ4IO_displayCompressedFilesInfo(const char** inFileNames, size_t ifnIdx)
if (op_result != LZ4IO_LZ4F_OK) {
assert(op_result == LZ4IO_format_not_known);
DISPLAYLEVEL(1, "lz4: %s: File format not recognized \n", inFileNames[idx]);
- return 0;
+ return 1;
} }
DISPLAYLEVEL(3, "\n");
if (g_displayLevel < 3) {
diff --git a/programs/lz4io.h b/programs/lz4io.h
index d6d7eee..0cfb1d2 100644
--- a/programs/lz4io.h
+++ b/programs/lz4io.h
@@ -1,6 +1,6 @@
/*
LZ4io.h - LZ4 File/Stream Interface
- Copyright (C) Yann Collet 2011-2016
+ Copyright (C) Yann Collet 2011-2020
GPL v2 License
This program is free software; you can redistribute it and/or modify
@@ -57,8 +57,6 @@ typedef struct LZ4IO_prefs_s LZ4IO_prefs_t;
LZ4IO_prefs_t* LZ4IO_defaultPreferences(void);
void LZ4IO_freePreferences(LZ4IO_prefs_t* prefs);
-/* Size in bytes of a legacy block header in little-endian format */
-#define LZIO_LEGACY_BLOCK_HEADER_SIZE 4
/* ************************************************** */
/* ****************** Functions ********************* */
diff --git a/programs/platform.h b/programs/platform.h
index ab8300d..43a171b 100644
--- a/programs/platform.h
+++ b/programs/platform.h
@@ -1,6 +1,6 @@
/*
platform.h - compiler and OS detection
- Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet
+ Copyright (C) 2016-2020, Przemyslaw Skibinski, Yann Collet
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -80,7 +80,7 @@ extern "C" {
************************************************************** */
#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \
|| defined(__midipix__) || defined(__VMS))
-# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1–2001 (SUSv3) conformant */ \
+# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1-2001 (SUSv3) conformant */ \
|| defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__MidnightBSD__) /* BSD distros */ \
|| defined(__HAIKU__)
# define PLATFORM_POSIX_VERSION 200112L
diff --git a/programs/util.h b/programs/util.h
index 733c1ca..3192ddc 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -1,6 +1,6 @@
/*
util.h - utility functions
- Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet
+ Copyright (C) 2016-2020, Przemyslaw Skibinski, Yann Collet
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -175,6 +175,39 @@ extern "C" {
#endif
+
+/*-****************************************
+* Allocation functions
+******************************************/
+/*
+ * A modified version of realloc().
+ * If UTIL_realloc() fails the original block is freed.
+*/
+UTIL_STATIC void* UTIL_realloc(void* ptr, size_t size)
+{
+ void* const newptr = realloc(ptr, size);
+ if (newptr) return newptr;
+ free(ptr);
+ return NULL;
+}
+
+
+/*-****************************************
+* String functions
+******************************************/
+/*
+ * A modified version of realloc().
+ * If UTIL_realloc() fails the original block is freed.
+*/
+UTIL_STATIC int UTIL_sameString(const char* a, const char* b)
+{
+ assert(a!=NULL && b!=NULL); /* unsupported scenario */
+ if (a==NULL) return 0;
+ if (b==NULL) return 0;
+ return !strcmp(a,b);
+}
+
+
/*-****************************************
* Time functions
******************************************/
@@ -317,6 +350,7 @@ UTIL_STATIC void UTIL_waitForNextTick(void)
UTIL_STATIC int UTIL_isRegFile(const char* infilename);
+UTIL_STATIC int UTIL_isRegFD(int fd);
UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf)
@@ -333,7 +367,8 @@ UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf)
timebuf.modtime = statbuf->st_mtime;
res += utime(filename, &timebuf); /* set access and modification times */
#else
- struct timespec timebuf[2] = {};
+ struct timespec timebuf[2];
+ memset(timebuf, 0, sizeof(timebuf));
timebuf[0].tv_nsec = UTIME_NOW;
timebuf[1].tv_sec = statbuf->st_mtime;
res += utimensat(AT_FDCWD, filename, timebuf, 0); /* set access and modification times */
@@ -351,6 +386,19 @@ UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf)
}
+UTIL_STATIC int UTIL_getFDStat(int fd, stat_t *statbuf)
+{
+ int r;
+#if defined(_MSC_VER)
+ r = _fstat64(fd, statbuf);
+ if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */
+#else
+ r = fstat(fd, statbuf);
+ if (r || !S_ISREG(statbuf->st_mode)) return 0; /* No good... */
+#endif
+ return 1;
+}
+
UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf)
{
int r;
@@ -365,6 +413,17 @@ UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf)
}
+UTIL_STATIC int UTIL_isRegFD(int fd)
+{
+ stat_t statbuf;
+#ifdef _WIN32
+ /* Windows runtime library always open file descriptors 0, 1 and 2 in text mode, therefore we can't use them for binary I/O */
+ if(fd < 3) return 0;
+#endif
+ return UTIL_getFDStat(fd, &statbuf); /* Only need to know whether it is a regular file */
+}
+
+
UTIL_STATIC int UTIL_isRegFile(const char* infilename)
{
stat_t statbuf;
@@ -425,19 +484,6 @@ UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFi
}
-/*
- * A modified version of realloc().
- * If UTIL_realloc() fails the original block is freed.
-*/
-UTIL_STATIC void* UTIL_realloc(void* ptr, size_t size)
-{
- void* const newptr = realloc(ptr, size);
- if (newptr) return newptr;
- free(ptr);
- return NULL;
-}
-
-
#ifdef _WIN32
# define UTIL_HAS_CREATEFILELIST
@@ -511,22 +557,23 @@ UTIL_STATIC int UTIL_prepareFileList(const char* dirName, char** bufStart, size_
{
DIR* dir;
struct dirent * entry;
- int dirLength, nbFiles = 0;
+ size_t dirLength;
+ int nbFiles = 0;
if (!(dir = opendir(dirName))) {
fprintf(stderr, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
return 0;
}
- dirLength = (int)strlen(dirName);
+ dirLength = strlen(dirName);
errno = 0;
while ((entry = readdir(dir)) != NULL) {
char* path;
- int fnameLength, pathLength;
+ size_t fnameLength, pathLength;
if (strcmp (entry->d_name, "..") == 0 ||
strcmp (entry->d_name, ".") == 0) continue;
- fnameLength = (int)strlen(entry->d_name);
- path = (char*) malloc(dirLength + fnameLength + 2);
+ fnameLength = strlen(entry->d_name);
+ path = (char*)malloc(dirLength + fnameLength + 2);
if (!path) { closedir(dir); return 0; }
memcpy(path, dirName, dirLength);
path[dirLength] = '/';
@@ -539,7 +586,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char* dirName, char** bufStart, size_
if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
} else {
if (*bufStart + *pos + pathLength >= *bufEnd) {
- ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
+ size_t const newListSize = (size_t)(*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
*bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
*bufEnd = *bufStart + newListSize;
if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
@@ -638,8 +685,8 @@ UTIL_createFileList(const char** inputNames, unsigned inputNamesNb,
UTIL_STATIC void
UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer)
{
- if (allocatedBuffer) free(allocatedBuffer);
- if (filenameTable) free((void*)filenameTable);
+ free(allocatedBuffer);
+ free((void*)filenameTable);
}
generated by cgit v0.12 at 2024-09-27 12:41:42 (GMT)