summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.travis.yml2
-rw-r--r--Makefile8
-rw-r--r--NEWS3
-rw-r--r--lib/lz4.c15
-rw-r--r--lib/lz4.h5
-rw-r--r--lib/lz4frame.c126
-rw-r--r--lib/lz4frame.h5
-rw-r--r--lib/lz4frame_static.h3
-rw-r--r--lib/lz4hc.c7
-rw-r--r--lib/lz4hc.h6
-rw-r--r--lib/xxhash.c3
-rw-r--r--programs/Makefile24
-rw-r--r--programs/bench.c15
-rw-r--r--programs/bench.h21
-rwxr-xr-x[-rw-r--r--]programs/datagen.c329
-rw-r--r--programs/datagen.h40
-rw-r--r--programs/datagencli.c193
-rw-r--r--programs/frametest.c11
-rw-r--r--[-rwxr-xr-x]programs/fullbench.c2
-rw-r--r--programs/fuzzer.c2
-rw-r--r--programs/lz4cli.c18
-rwxr-xr-x[-rw-r--r--]programs/lz4io.c206
-rw-r--r--programs/lz4io.h11
23 files changed, 648 insertions, 407 deletions
diff --git a/.travis.yml b/.travis.yml
index 9eff075..650fba1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,6 +3,7 @@ compiler: gcc
script: make test-travis
before_install:
- sudo apt-get update -qq
+ - sudo apt-get install -qq g++-multilib
- sudo apt-get install -qq gcc-multilib
- sudo apt-get install -qq valgrind
@@ -10,6 +11,7 @@ env:
- LZ4_TRAVIS_CI_ENV=travis-install
- LZ4_TRAVIS_CI_ENV=streaming-examples
- LZ4_TRAVIS_CI_ENV=cmake
+ - LZ4_TRAVIS_CI_ENV=gpptest
- LZ4_TRAVIS_CI_ENV=dist
- LZ4_TRAVIS_CI_ENV=test-lz4
- LZ4_TRAVIS_CI_ENV=test-lz4c
diff --git a/Makefile b/Makefile
index 5662cb4..b10e4d7 100644
--- a/Makefile
+++ b/Makefile
@@ -50,7 +50,7 @@ TEXT = $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4.h $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4hc.h \
Makefile lz4_block_format.txt LZ4_Frame_Format.html NEWS README.md \
cmake_unofficial/CMakeLists.txt \
$(PRGDIR)/fullbench.c $(PRGDIR)/lz4cli.c \
- $(PRGDIR)/datagen.c $(PRGDIR)/fuzzer.c \
+ $(PRGDIR)/datagen.c $(PRGDIR)/datagen.h $(PRGDIR)/datagencli.c $(PRGDIR)/fuzzer.c \
$(PRGDIR)/lz4io.c $(PRGDIR)/lz4io.h \
$(PRGDIR)/bench.c $(PRGDIR)/bench.h \
$(PRGDIR)/lz4.1 $(PRGDIR)/lz4c.1 $(PRGDIR)/lz4cat.1 \
@@ -82,7 +82,7 @@ clean:
@rm -f $(DISTRIBNAME) *.sha1
@cd $(PRGDIR); $(MAKE) clean
@cd $(LZ4DIR); $(MAKE) clean
- @cd examples; $(MAKE) clean
+ @cd examples; $(MAKE) clean
@echo Cleaning completed
@@ -127,6 +127,10 @@ test-travis: $(TRAVIS_TARGET)
cmake:
@cd cmake_unofficial; cmake CMakeLists.txt; $(MAKE)
+gpptest:
+ export CC=g++; export CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align"; $(MAKE) -e all
+
+
streaming-examples:
cd examples; $(MAKE) -e test
diff --git a/NEWS b/NEWS
index dbea778..09247c0 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,9 @@
r128:
+New : lz4 cli sparse file support
New : command -m, to compress multiple files in a single command
Fixed : Restored lz4hc compression ratio (slightly lower since r124)
+New : g++ compatibility test
+New : datagen can generate sparse files
Fixed : Fuzzer + frametest compatibility with NetBSD (issue #48)
Added : Visual project directory
diff --git a/lib/lz4.c b/lib/lz4.c
index a8b4531..a651843 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -27,8 +27,7 @@
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- - LZ4 source repository : http://code.google.com/p/lz4
- - LZ4 source mirror : https://github.com/Cyan4973/lz4
+ - LZ4 source repository : https://github.com/Cyan4973/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
@@ -190,7 +189,7 @@ static U16 LZ4_readLE16(const void* memPtr)
return *(U16*)memPtr;
else
{
- const BYTE* p = memPtr;
+ const BYTE* p = (const BYTE*)memPtr;
return (U16)((U16)p[0] + (p[1]<<8));
}
}
@@ -204,7 +203,7 @@ static void LZ4_writeLE16(void* memPtr, U16 value)
}
else
{
- BYTE* p = memPtr;
+ BYTE* p = (BYTE*)memPtr;
p[0] = (BYTE) value;
p[1] = (BYTE)(value>>8);
}
@@ -285,9 +284,9 @@ static void LZ4_copy8(void* dstPtr, const void* srcPtr)
/* customized version of memcpy, which may overwrite up to 7 bytes beyond dstEnd */
static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd)
{
- BYTE* d = dstPtr;
- const BYTE* s = srcPtr;
- BYTE* e = dstEnd;
+ BYTE* d = (BYTE*)dstPtr;
+ const BYTE* s = (const BYTE*)srcPtr;
+ BYTE* e = (BYTE*)dstEnd;
do { LZ4_copy8(d,s); d+=8; s+=8; } while (d<e);
}
@@ -1151,7 +1150,7 @@ typedef struct
*/
LZ4_streamDecode_t* LZ4_createStreamDecode(void)
{
- LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(sizeof(U64), LZ4_STREAMDECODESIZE_U64);
+ LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(LZ4_STREAMDECODESIZE_U64, sizeof(U64));
return lz4s;
}
diff --git a/lib/lz4.h b/lib/lz4.h
index 7778caa..df1d839 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -1,7 +1,8 @@
/*
LZ4 - Fast LZ compression algorithm
Header File
- Copyright (C) 2011-2014, Yann Collet.
+ Copyright (C) 2011-2015, Yann Collet.
+
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
@@ -28,7 +29,7 @@
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- - LZ4 source repository : http://code.google.com/p/lz4/
+ - LZ4 source repository : https://github.com/Cyan4973/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
#pragma once
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 5183f22..7153871 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -1,6 +1,7 @@
/*
LZ4 auto-framing library
-Copyright (C) 2011-2014, Yann Collet.
+Copyright (C) 2011-2015, Yann Collet.
+
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
@@ -27,13 +28,13 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
-- LZ4 source repository : http://code.google.com/p/lz4/
+- LZ4 source repository : https://github.com/Cyan4973/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
/* LZ4F is a stand-alone API to create LZ4-compressed Frames
-* fully conformant to specification v1.4.1.
-* All related operations, including memory management, are handled by the library.
+* in full conformance with specification v1.4.1.
+* All related operations, including memory management, are handled by the library.
* */
@@ -45,14 +46,10 @@ Compiler Options
#endif
#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#ifdef __GNUC__
-# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
-# pragma GCC diagnostic ignored "-Wmissing-field-initializers" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
-#endif
/**************************************
-Memory routines
+* Memory routines
**************************************/
#include <stdlib.h> /* malloc, calloc, free */
#define ALLOCATOR(s) calloc(1,s)
@@ -62,7 +59,7 @@ Memory routines
/**************************************
-Includes
+* Includes
**************************************/
#include "lz4frame_static.h"
#include "lz4.h"
@@ -71,7 +68,7 @@ Includes
/**************************************
-Basic Types
+* Basic Types
**************************************/
#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
# include <stdint.h>
@@ -90,7 +87,7 @@ typedef unsigned long long U64;
/**************************************
-Constants
+* Constants
**************************************/
#define KB *(1<<10)
#define MB *(1<<20)
@@ -110,7 +107,7 @@ Constants
static const U32 minHClevel = 3;
/**************************************
-Structures and local types
+* Structures and local types
**************************************/
typedef struct
{
@@ -150,12 +147,7 @@ typedef struct
/**************************************
-Macros
-**************************************/
-
-
-/**************************************
-Error management
+* Error management
**************************************/
#define LZ4F_GENERATE_STRING(STRING) #STRING,
static const char* LZ4F_errorStrings[] = { LZ4F_LIST_ERRORS(LZ4F_GENERATE_STRING) };
@@ -175,7 +167,7 @@ const char* LZ4F_getErrorName(LZ4F_errorCode_t code)
/**************************************
-Private functions
+* Private functions
**************************************/
static size_t LZ4F_getBlockSize(unsigned blockSizeID)
{
@@ -215,14 +207,15 @@ static BYTE LZ4F_headerChecksum (const BYTE* header, size_t length)
/**************************************
-Simple compression functions
+* Simple compression functions
**************************************/
size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
{
- LZ4F_preferences_t prefs = { 0 };
+ LZ4F_preferences_t prefs;
size_t headerSize;
size_t streamSize;
+ memset(&prefs, 0, sizeof(prefs));
if (preferencesPtr!=NULL) prefs = *preferencesPtr;
{
blockSizeID_t proposedBSID = max64KB;
@@ -234,7 +227,7 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere
prefs.frameInfo.blockSizeID = proposedBSID;
break;
}
- proposedBSID++;
+ proposedBSID = (blockSizeID_t)( ((int)proposedBSID) + 1);
maxBlockSize <<= 2;
}
}
@@ -258,14 +251,17 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere
*/
size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
{
- LZ4F_cctx_internal_t cctxI = { 0 }; /* works because no allocation */
- LZ4F_preferences_t prefs = { 0 };
- LZ4F_compressOptions_t options = { 0 };
+ LZ4F_cctx_internal_t cctxI;
+ LZ4F_preferences_t prefs;
+ LZ4F_compressOptions_t options;
LZ4F_errorCode_t errorCode;
BYTE* const dstStart = (BYTE*) dstBuffer;
BYTE* dstPtr = dstStart;
BYTE* const dstEnd = dstStart + dstMaxSize;
+ memset(&cctxI, 0, sizeof(cctxI)); /* works because no allocation */
+ memset(&prefs, 0, sizeof(prefs));
+ memset(&options, 0, sizeof(options));
cctxI.version = LZ4F_VERSION;
cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent allocation; works because autoflush==1 & stableSrc==1 */
@@ -281,7 +277,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf
prefs.frameInfo.blockSizeID = proposedBSID;
break;
}
- proposedBSID++;
+ proposedBSID = (blockSizeID_t)((int)proposedBSID + 1);
maxBlockSize <<= 2;
}
}
@@ -361,7 +357,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
*/
size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_preferences_t* preferencesPtr)
{
- LZ4F_preferences_t prefNull = { 0 };
+ LZ4F_preferences_t prefNull;
LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext;
BYTE* const dstStart = (BYTE*)dstBuffer;
BYTE* dstPtr = dstStart;
@@ -370,6 +366,7 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds
if (dstMaxSize < LZ4F_MAXHEADERFRAME_SIZE) return (size_t)-ERROR_dstMaxSize_tooSmall;
if (cctxPtr->cStage != 0) return (size_t)-ERROR_GENERIC;
+ memset(&prefNull, 0, sizeof(prefNull));
if (preferencesPtr == NULL) preferencesPtr = &prefNull;
cctxPtr->prefs = *preferencesPtr;
@@ -436,17 +433,19 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds
* */
size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
{
- const LZ4F_preferences_t prefsNull = { 0 };
- const LZ4F_preferences_t* prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr;
- blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID;
- size_t blockSize = LZ4F_getBlockSize(bid);
- unsigned nbBlocks = (unsigned)(srcSize / blockSize) + 1;
- size_t lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize;
- size_t blockInfo = 4; /* default, without block CRC option */
- size_t frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4);
- size_t result = (blockInfo * nbBlocks) + (blockSize * (nbBlocks-1)) + lastBlockSize + frameEnd;
-
- return result;
+ LZ4F_preferences_t prefsNull;
+ memset(&prefsNull, 0, sizeof(prefsNull));
+ {
+ const LZ4F_preferences_t* prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr;
+ blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID;
+ size_t blockSize = LZ4F_getBlockSize(bid);
+ unsigned nbBlocks = (unsigned)(srcSize / blockSize) + 1;
+ size_t lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize;
+ size_t blockInfo = 4; /* default, without block CRC option */
+ size_t frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4);
+
+ return (blockInfo * nbBlocks) + (blockSize * (nbBlocks-1)) + lastBlockSize + frameEnd;;
+ }
}
@@ -518,7 +517,7 @@ typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
*/
size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr)
{
- LZ4F_compressOptions_t cOptionsNull = { 0 };
+ LZ4F_compressOptions_t cOptionsNull;
LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext;
size_t blockSize = cctxPtr->maxBlockSize;
const BYTE* srcPtr = (const BYTE*)srcBuffer;
@@ -531,6 +530,7 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d
if (cctxPtr->cStage != 1) return (size_t)-ERROR_GENERIC;
if (dstMaxSize < LZ4F_compressBound(srcSize, &(cctxPtr->prefs))) return (size_t)-ERROR_dstMaxSize_tooSmall;
+ memset(&cOptionsNull, 0, sizeof(cOptionsNull));
if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull;
/* select compression function */
@@ -627,7 +627,7 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d
*/
size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr)
{
- LZ4F_compressOptions_t cOptionsNull = { 0 };
+ LZ4F_compressOptions_t cOptionsNull;
LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext;
BYTE* const dstStart = (BYTE*)dstBuffer;
BYTE* dstPtr = dstStart;
@@ -637,8 +637,8 @@ size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer,
if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */
if (cctxPtr->cStage != 1) return (size_t)-ERROR_GENERIC;
if (dstMaxSize < (cctxPtr->tmpInSize + 16)) return (size_t)-ERROR_dstMaxSize_tooSmall;
+ memset(&cOptionsNull, 0, sizeof(cOptionsNull));
if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull;
- (void)compressOptionsPtr; /* not yet useful */
/* select compression function */
compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
@@ -712,7 +712,7 @@ LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_compressionContext_t* LZ4F
{
LZ4F_dctx_internal_t* dctxPtr;
- dctxPtr = ALLOCATOR(sizeof(LZ4F_dctx_internal_t));
+ dctxPtr = (LZ4F_dctx_internal_t*)ALLOCATOR(sizeof(LZ4F_dctx_internal_t));
if (dctxPtr==NULL) return (LZ4F_errorCode_t)-ERROR_GENERIC;
dctxPtr->version = versionNumber;
@@ -732,11 +732,12 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_compressionContext_t LZ4F_de
/* Decompression */
-static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPtr, size_t srcSize)
+static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const void* srcVoidPtr, size_t srcSize)
{
BYTE FLG, BD, HC;
unsigned version, blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, dictFlag, blockSizeID;
size_t bufferNeeded;
+ const BYTE* srcPtr = (const BYTE*)srcVoidPtr;
/* need to decode header to get frameInfo */
if (srcSize < 7) return (size_t)-ERROR_GENERIC; /* minimal header size */
@@ -747,7 +748,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPt
/* Flags */
FLG = srcPtr[0];
- version = (FLG>>6)&_2BITS;
+ version = (FLG>>6) & _2BITS;
blockMode = (FLG>>5) & _1BIT;
blockChecksumFlag = (FLG>>4) & _1BIT;
contentSizeFlag = (FLG>>3) & _1BIT;
@@ -761,19 +762,19 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPt
if (HC != srcPtr[2]) return (size_t)-ERROR_GENERIC; /* Bad header checksum error */
/* validate */
- if (version != 1) return (size_t)-ERROR_GENERIC; /* Version Number, only supported value */
- if (blockChecksumFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */
+ if (version != 1) return (size_t)-ERROR_GENERIC; /* Version Number, only supported value */
+ if (blockChecksumFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */
if (contentSizeFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */
- if (((FLG>>1)&_1BIT) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bit */
- if (dictFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */
+ if (((FLG>>1)&_1BIT) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bit */
+ if (dictFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */
if (((BD>>7)&_1BIT) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bit */
- if (blockSizeID < 4) return (size_t)-ERROR_GENERIC; /* Only supported values for the time being */
- if (((BD>>0)&_4BITS) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bits */
+ if (blockSizeID < 4) return (size_t)-ERROR_GENERIC; /* 4-7 only supported values for the time being */
+ if (((BD>>0)&_4BITS) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bits */
/* save */
- dctxPtr->frameInfo.blockMode = blockMode;
- dctxPtr->frameInfo.contentChecksumFlag = contentChecksumFlag;
- dctxPtr->frameInfo.blockSizeID = blockSizeID;
+ dctxPtr->frameInfo.blockMode = (blockMode_t)blockMode;
+ dctxPtr->frameInfo.contentChecksumFlag = (contentChecksum_t)contentChecksumFlag;
+ dctxPtr->frameInfo.blockSizeID = (blockSizeID_t)blockSizeID;
dctxPtr->maxBlockSize = LZ4F_getBlockSize(blockSizeID);
/* init */
@@ -786,9 +787,9 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPt
FREEMEM(dctxPtr->tmpIn);
FREEMEM(dctxPtr->tmpOutBuffer);
dctxPtr->maxBufferSize = bufferNeeded;
- dctxPtr->tmpIn = ALLOCATOR(dctxPtr->maxBlockSize);
+ dctxPtr->tmpIn = (BYTE*)ALLOCATOR(dctxPtr->maxBlockSize);
if (dctxPtr->tmpIn == NULL) return (size_t)-ERROR_GENERIC;
- dctxPtr->tmpOutBuffer= ALLOCATOR(dctxPtr->maxBufferSize);
+ dctxPtr->tmpOutBuffer= (BYTE*)ALLOCATOR(dctxPtr->maxBufferSize);
if (dctxPtr->tmpOutBuffer== NULL) return (size_t)-ERROR_GENERIC;
}
dctxPtr->tmpInSize = 0;
@@ -880,15 +881,6 @@ static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, s
if (withinTmp) /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */
{
-#if 0
- size_t savedDictSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer;
- memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart- savedDictSize, savedDictSize);
- dctxPtr->dict = dctxPtr->tmpOutBuffer;
- dctxPtr->dictSize = savedDictSize + dctxPtr->tmpOutStart + dstSize;
- return;
-
-#else
-
size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer;
size_t copySize = 64 KB - dctxPtr->tmpOutSize;
BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart;
@@ -900,7 +892,6 @@ static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, s
dctxPtr->dict = dctxPtr->tmpOutBuffer;
dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart + dstSize;
return;
-#endif
}
if (dctxPtr->dict == dctxPtr->tmpOutBuffer) /* copy dst into tmp to complete dict */
@@ -952,7 +943,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
const LZ4F_decompressOptions_t* decompressOptionsPtr)
{
LZ4F_dctx_internal_t* dctxPtr = (LZ4F_dctx_internal_t*)decompressionContext;
- static const LZ4F_decompressOptions_t optionsNull = { 0 };
+ LZ4F_decompressOptions_t optionsNull;
const BYTE* const srcStart = (const BYTE*)srcBuffer;
const BYTE* const srcEnd = srcStart + *srcSizePtr;
const BYTE* srcPtr = srcStart;
@@ -964,6 +955,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
size_t nextSrcSizeHint = 1;
+ memset(&optionsNull, 0, sizeof(optionsNull));
if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull;
*srcSizePtr = 0;
*dstSizePtr = 0;
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index d73e3e2..61e461b 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -28,8 +28,7 @@
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- - LZ4 source repository : http://code.google.com/p/lz4/
- - LZ4 source mirror : https://github.com/Cyan4973/lz4
+ - LZ4 source repository : https://github.com/Cyan4973/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
@@ -215,7 +214,7 @@ size_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t ctx,
* The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
* You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr)
* The function result is an hint of how many srcSize bytes LZ4F_decompress() expects for next call,
- * or an error code which can be tested using LZ4F_isError().
+ * or an error code which can be tested using LZ4F_isError().
*/
size_t LZ4F_decompress(LZ4F_decompressionContext_t ctx,
diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h
index cde8186..4c34c6c 100644
--- a/lib/lz4frame_static.h
+++ b/lib/lz4frame_static.h
@@ -29,8 +29,7 @@
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- - LZ4 source repository : http://code.google.com/p/lz4/
- - LZ4 source mirror : https://github.com/Cyan4973/lz4
+ - LZ4 source repository : https://github.com/Cyan4973/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 5549969..357fa96 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -1,6 +1,7 @@
/*
LZ4 HC - High Compression Mode of LZ4
-Copyright (C) 2011-2014, Yann Collet.
+Copyright (C) 2011-2015, Yann Collet.
+
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
@@ -27,8 +28,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
-- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
-- LZ4 source repository : http://code.google.com/p/lz4/
+ - LZ4 source repository : https://github.com/Cyan4973/lz4
+ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index ce813ab..eb72051 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -1,7 +1,7 @@
/*
LZ4 HC - High Compression Mode of LZ4
Header File
- Copyright (C) 2011-2014, Yann Collet.
+ Copyright (C) 2011-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
@@ -28,8 +28,8 @@
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- - LZ4 source repository : http://code.google.com/p/lz4/
+ - LZ4 source repository : https://github.com/Cyan4973/lz4
+ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
#pragma once
diff --git a/lib/xxhash.c b/lib/xxhash.c
index 093564c..aca1e0a 100644
--- a/lib/xxhash.c
+++ b/lib/xxhash.c
@@ -28,8 +28,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
-- xxHash source repository : http://code.google.com/p/xxhash/
-- xxHash source mirror : https://github.com/Cyan4973/xxHash
+- xxHash source repository : https://github.com/Cyan4973/xxHash
- public discussion board : https://groups.google.com/forum/#!forum/lz4c
*/
diff --git a/programs/Makefile b/programs/Makefile
index 1070f40..3d38244 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -92,7 +92,7 @@ frametest: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxha
frametest32: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c frametest.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
-datagen : datagen.c
+datagen : datagen.c datagencli.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
@@ -142,11 +142,11 @@ test-travis: $(TRAVIS_TARGET)
test-lz4: lz4 datagen
./datagen -g16KB | ./lz4 -9 | ./lz4 -vdq > $(VOID)
./datagen | ./lz4 | ./lz4 -vdq > $(VOID)
- ./datagen -g6M -p100 | ./lz4 -9BD | ./lz4 -vdq > $(VOID)
- ./datagen -g17M | ./lz4 -9v | ./lz4 -vdq > $(VOID)
+ ./datagen -g6M -P100 | ./lz4 -9BD | ./lz4 -vdq > $(VOID)
+ ./datagen -g17M | ./lz4 -9v | ./lz4 -dq > $(VOID)
./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -vdq > $(VOID)
./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -vdq > $(VOID)
-# test frame concatenation with null-length frame
+ @echo ---- test frame concatenation ----
@echo -n > empty.test
@echo hi > nonempty.test
cat nonempty.test empty.test nonempty.test > orig.test
@@ -157,13 +157,24 @@ test-lz4: lz4 datagen
sdiff orig.test result.test
@rm *.test
@echo frame concatenation test completed
-# test frame concatenation with null-length frame
- @echo test multiple input files
+ @echo ---- test multiple input files ----
@./datagen -s1 > file1
@./datagen -s2 > file2
@./datagen -s3 > file3
./lz4 -f -m file1 file2 file3
+ ls -l file*
@rm file1 file2 file3 file1.lz4 file2.lz4 file3.lz4
+ @echo ---- test sparse file support ----
+ ./datagen -g50M -P100 | ./lz4 -B4D | ./lz4 -dvX > tmpB4
+ ./datagen -g50M -P100 | ./lz4 -B5D | ./lz4 -dvX > tmpB5
+ ./datagen -g50M -P100 | ./lz4 -B6D | ./lz4 -dvX > tmpB6
+ ./datagen -g50M -P100 | ./lz4 -B7D | ./lz4 -dvX > tmpB7
+ ls -ls tmp*
+ ./datagen -g50M -P100 | diff -s - tmpB4
+ ./datagen -g50M -P100 | diff -s - tmpB5
+ ./datagen -g50M -P100 | diff -s - tmpB6
+ ./datagen -g50M -P100 | diff -s - tmpB7
+ @rm tmp*
test-lz4c: lz4c datagen
@@ -197,6 +208,7 @@ test-frametest32: frametest32
./frametest32
test-mem: lz4 datagen fuzzer frametest
+ valgrind --leak-check=yes ./datagen -g50M > /dev/null
./datagen -g16KB > tmp
valgrind --leak-check=yes ./lz4 -9 -BD -f tmp /dev/null
./datagen -g16MB > tmp
diff --git a/programs/bench.c b/programs/bench.c
index 77120f2..b632314 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -1,6 +1,7 @@
/*
- bench.c - Demo program to benchmark open-source compression algorithm
+ bench.c - Demo program to benchmark open-source compression algorithms
Copyright (C) Yann Collet 2012-2015
+
GPL v2 License
This program is free software; you can redistribute it and/or modify
@@ -18,8 +19,8 @@
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- - LZ4 source repository : http://code.google.com/p/lz4/
+ - LZ4 source repository : https://github.com/Cyan4973/lz4
+ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
/**************************************
@@ -141,15 +142,15 @@ static int chunkSize = DEFAULT_CHUNKSIZE;
static int nbIterations = NBLOOPS;
static int BMK_pause = 0;
-void BMK_SetBlocksize(int bsize) { chunkSize = bsize; }
+void BMK_setBlocksize(int bsize) { chunkSize = bsize; }
-void BMK_SetNbIterations(int nbLoops)
+void BMK_setNbIterations(int nbLoops)
{
nbIterations = nbLoops;
DISPLAY("- %i iterations -\n", nbIterations);
}
-void BMK_SetPause(void) { BMK_pause = 1; }
+void BMK_setPause(void) { BMK_pause = 1; }
/*********************************************************
@@ -234,7 +235,7 @@ static U64 BMK_GetFileSize(const char* infilename)
* Public function
**********************************************************/
-int BMK_benchFile(const char** fileNamesTable, int nbFiles, int cLevel)
+int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel)
{
int fileIdx=0;
char* orig_buff;
diff --git a/programs/bench.h b/programs/bench.h
index 2a20cdb..c04fb17 100644
--- a/programs/bench.h
+++ b/programs/bench.h
@@ -17,26 +17,17 @@
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- - LZ4 source repository : http://code.google.com/p/lz4/
- - LZ4 public forum : https://group.google.com/forum/#!forum/lz4c
+ - LZ4 source repository : https://github.com/Cyan4973/lz4
+ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
#pragma once
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
/* Main function */
-int BMK_benchFile(const char** fileNamesTable, int nbFiles, int cLevel);
+int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel);
/* Set Parameters */
-void BMK_SetBlocksize(int bsize);
-void BMK_SetNbIterations(int nbLoops);
-void BMK_SetPause(void);
-
-
+void BMK_setBlocksize(int bsize);
+void BMK_setNbIterations(int nbLoops);
+void BMK_setPause(void);
-#if defined (__cplusplus)
-}
-#endif
diff --git a/programs/datagen.c b/programs/datagen.c
index 0f07477..bccb21e 100644..100755
--- a/programs/datagen.c
+++ b/programs/datagen.c
@@ -19,26 +19,20 @@
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- - LZ4 source repository : http://code.google.com/p/lz4
- - LZ4 source mirror : https://github.com/Cyan4973/lz4
- - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
+ - ZSTD source repository : https://github.com/Cyan4973/zstd
+ - Public forum : https://groups.google.com/forum/#!forum/lz4c
*/
/**************************************
- Remove Visual warning messages
+* Includes
**************************************/
-#define _CRT_SECURE_NO_WARNINGS // fgets
+#include <stdlib.h> /* malloc */
+#include <stdio.h> /* FILE, fwrite */
+#include <string.h> /* memcpy */
/**************************************
- Includes
-**************************************/
-#include <stdio.h> // fgets, sscanf
-#include <string.h> // strcmp
-
-
-/**************************************
- Basic Types
+* Basic Types
**************************************/
#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
# include <stdint.h>
@@ -57,230 +51,173 @@
/**************************************
- Constants
+* OS-specific Includes
**************************************/
-#ifndef LZ4_VERSION
-# define LZ4_VERSION "r125"
+#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
+# include <fcntl.h> /* _O_BINARY */
+# include <io.h> /* _setmode, _isatty */
+# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
+#else
+# define SET_BINARY_MODE(file)
#endif
-#define KB *(1 <<10)
-#define MB *(1 <<20)
-#define GB *(1U<<30)
-
-#define CDG_SIZE_DEFAULT (64 KB)
-#define CDG_SEED_DEFAULT 0
-#define CDG_COMPRESSIBILITY_DEFAULT 50
-#define PRIME1 2654435761U
-#define PRIME2 2246822519U
-
/**************************************
- Macros
+* Constants
**************************************/
-#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
-#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
-
+#define KB *(1 <<10)
-/**************************************
- Local Parameters
-**************************************/
-static unsigned no_prompt = 0;
-static char* programName;
-static unsigned displayLevel = 2;
+#define PRIME1 2654435761U
+#define PRIME2 2246822519U
/*********************************************************
- functions
+* Local Functions
*********************************************************/
-
-#define CDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
-static unsigned int CDG_rand(U32* src)
+#define RDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+static unsigned int RDG_rand(U32* src)
{
U32 rand32 = *src;
rand32 *= PRIME1;
- rand32 += PRIME2;
- rand32 = CDG_rotl32(rand32, 13);
+ rand32 ^= PRIME2;
+ rand32 = RDG_rotl32(rand32, 13);
*src = rand32;
return rand32;
}
-#define CDG_RAND15BITS ((CDG_rand(seed) >> 3) & 32767)
-#define CDG_RANDLENGTH ( ((CDG_rand(seed) >> 7) & 3) ? (CDG_rand(seed) % 14) : (CDG_rand(seed) & 511) + 15)
-#define CDG_RANDCHAR (((CDG_rand(seed) >> 9) & 63) + '0')
-static void CDG_generate(U64 size, U32* seed, double proba)
+#define LTSIZE 8192
+#define LTMASK (LTSIZE-1)
+static void* RDG_createLiteralDistrib(double ld)
{
- BYTE fullbuff[32 KB + 128 KB + 1];
- BYTE* buff = fullbuff + 32 KB;
- U64 total=0;
- U32 P32 = (U32)(32768 * proba);
- U32 pos=1;
- U32 genBlockSize = 128 KB;
+ BYTE* lt = (BYTE*)malloc(LTSIZE);
+ U32 i = 0;
+ BYTE character = '0';
+ BYTE firstChar = '(';
+ BYTE lastChar = '}';
- // Build initial prefix
- fullbuff[0] = CDG_RANDCHAR;
- while (pos<32 KB)
+ if (ld==0.0)
{
- // Select : Literal (char) or Match (within 32K)
- if (CDG_RAND15BITS < P32)
- {
- // Copy (within 64K)
- U32 d;
- int ref;
- int length = CDG_RANDLENGTH + 4;
- U32 offset = CDG_RAND15BITS + 1;
- if (offset > pos) offset = pos;
- ref = pos - offset;
- d = pos + length;
- while (pos < d) fullbuff[pos++] = fullbuff[ref++];
- }
- else
+ character = 0;
+ firstChar = 0;
+ lastChar =255;
+ }
+ while (i<LTSIZE)
+ {
+ U32 weight = (U32)((double)(LTSIZE - i) * ld) + 1;
+ U32 end;
+ if (weight + i > LTSIZE) weight = LTSIZE-i;
+ end = i + weight;
+ while (i < end) lt[i++] = character;
+ character++;
+ if (character > lastChar) character = firstChar;
+ }
+ return lt;
+}
+
+static char RDG_genChar(U32* seed, const void* ltctx)
+{
+ const BYTE* lt = (const BYTE*)ltctx;
+ U32 id = RDG_rand(seed) & LTMASK;
+ return lt[id];
+}
+
+#define RDG_DICTSIZE (32 KB)
+#define RDG_RAND15BITS ((RDG_rand(seed) >> 3) & 32767)
+#define RDG_RANDLENGTH ( ((RDG_rand(seed) >> 7) & 7) ? (RDG_rand(seed) & 15) : (RDG_rand(seed) & 511) + 15)
+void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, void* litTable, unsigned* seedPtr)
+{
+ BYTE* buffPtr = (BYTE*)buffer;
+ const U32 matchProba32 = (U32)(32768 * matchProba);
+ size_t pos = prefixSize;
+ void* ldctx = litTable;
+ U32* seed = seedPtr;
+
+ /* special case */
+ while (matchProba >= 1.0)
+ {
+ size_t size0 = RDG_rand(seed) & 3;
+ size0 = (size_t)1 << (16 + size0 * 2);
+ size0 += RDG_rand(seed) & (size0-1); /* because size0 is power of 2*/
+ if (buffSize < pos + size0)
{
- // Literal (noise)
- U32 d = pos + CDG_RANDLENGTH;
- while (pos < d) fullbuff[pos++] = CDG_RANDCHAR;
+ memset(buffPtr+pos, 0, buffSize-pos);
+ return;
}
+ memset(buffPtr+pos, 0, size0);
+ pos += size0;
+ buffPtr[pos-1] = RDG_genChar(seed, ldctx);
}
- // Generate compressible data
- pos = 0;
- while (total < size)
+ /* init */
+ if (pos==0) buffPtr[0] = RDG_genChar(seed, ldctx), pos=1;
+
+ /* Generate compressible data */
+ while (pos < buffSize)
{
- if (size-total < 128 KB) genBlockSize = (U32)(size-total);
- total += genBlockSize;
- buff[genBlockSize] = 0;
- pos = 0;
- while (pos<genBlockSize)
+ /* Select : Literal (char) or Match (within 32K) */
+ if (RDG_RAND15BITS < matchProba32)
+ {
+ /* Copy (within 32K) */
+ size_t match;
+ size_t d;
+ int length = RDG_RANDLENGTH + 4;
+ U32 offset = RDG_RAND15BITS + 1;
+ if (offset > pos) offset = (U32)pos;
+ match = pos - offset;
+ d = pos + length;
+ if (d > buffSize) d = buffSize;
+ while (pos < d) buffPtr[pos++] = buffPtr[match++];
+ }
+ else
{
- // Select : Literal (char) or Match (within 32K)
- if (CDG_RAND15BITS < P32)
- {
- // Copy (within 64K)
- int ref;
- U32 d;
- int length = CDG_RANDLENGTH + 4;
- U32 offset = CDG_RAND15BITS + 1;
- if (pos + length > genBlockSize ) length = genBlockSize - pos;
- ref = pos - offset;
- d = pos + length;
- while (pos < d) buff[pos++] = buff[ref++];
- }
- else
- {
- // Literal (noise)
- U32 d;
- int length = CDG_RANDLENGTH;
- if (pos + length > genBlockSize) length = genBlockSize - pos;
- d = pos + length;
- while (pos < d) buff[pos++] = CDG_RANDCHAR;
- }
+ /* Literal (noise) */
+ size_t d;
+ size_t length = RDG_RANDLENGTH;
+ d = pos + length;
+ if (d > buffSize) d = buffSize;
+ while (pos < d) buffPtr[pos++] = RDG_genChar(seed, ldctx);
}
- // output datagen
- pos=0;
- for (;pos+512<=genBlockSize;pos+=512)
- printf("%512.512s", buff+pos);
- for (;pos<genBlockSize;pos++) printf("%c", buff[pos]);
- // Regenerate prefix
- memcpy(fullbuff, buff + 96 KB, 32 KB);
}
}
-int CDG_usage(void)
+void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed)
{
- DISPLAY( "Compressible data generator\n");
- DISPLAY( "Usage :\n");
- DISPLAY( " %s [size] [args]\n", programName);
- DISPLAY( "\n");
- DISPLAY( "Arguments :\n");
- DISPLAY( " -g# : generate # data (default:%i)\n", CDG_SIZE_DEFAULT);
- DISPLAY( " -s# : Select seed (default:%i)\n", CDG_SEED_DEFAULT);
- DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", CDG_COMPRESSIBILITY_DEFAULT);
- DISPLAY( " -h : display help and exit\n");
- return 0;
+ void* ldctx;
+ if (litProba==0.0) litProba = matchProba / 4.5;
+ ldctx = RDG_createLiteralDistrib(litProba);
+ RDG_genBlock(buffer, size, 0, matchProba, ldctx, &seed);
+ free(ldctx);
}
-int main(int argc, char** argv)
+#define RDG_BLOCKSIZE (128 KB)
+void RDG_genOut(unsigned long long size, double matchProba, double litProba, unsigned seed)
{
- int argNb;
- int proba = CDG_COMPRESSIBILITY_DEFAULT;
- U64 size = CDG_SIZE_DEFAULT;
- U32 seed = CDG_SEED_DEFAULT;
-
- // Check command line
- programName = argv[0];
- for(argNb=1; argNb<argc; argNb++)
- {
- char* argument = argv[argNb];
-
- if(!argument) continue; // Protection if argument empty
+ BYTE buff[RDG_DICTSIZE + RDG_BLOCKSIZE];
+ U64 total = 0;
+ size_t genBlockSize = RDG_BLOCKSIZE;
+ void* ldctx;
- // Decode command (note : aggregated commands are allowed)
- if (*argument=='-')
- {
- if (!strcmp(argument, "--no-prompt")) { no_prompt=1; continue; }
+ /* init */
+ if (litProba==0.0) litProba = matchProba / 4.5;
+ ldctx = RDG_createLiteralDistrib(litProba);
+ SET_BINARY_MODE(stdout);
- argument++;
- while (*argument!=0)
- {
- switch(*argument)
- {
- case 'h':
- return CDG_usage();
- case 'g':
- argument++;
- size=0;
- while ((*argument>='0') && (*argument<='9'))
- {
- size *= 10;
- size += *argument - '0';
- argument++;
- }
- if (*argument=='K') { size <<= 10; argument++; }
- if (*argument=='M') { size <<= 20; argument++; }
- if (*argument=='G') { size <<= 30; argument++; }
- if (*argument=='B') { argument++; }
- break;
- case 's':
- argument++;
- seed=0;
- while ((*argument>='0') && (*argument<='9'))
- {
- seed *= 10;
- seed += *argument - '0';
- argument++;
- }
- break;
- case 'p':
- argument++;
- proba=0;
- while ((*argument>='0') && (*argument<='9'))
- {
- proba *= 10;
- proba += *argument - '0';
- argument++;
- }
- if (proba<0) proba=0;
- if (proba>100) proba=100;
- break;
- case 'v':
- displayLevel = 4;
- argument++;
- break;
- default: ;
- }
- }
+ /* Generate dict */
+ RDG_genBlock(buff, RDG_DICTSIZE, 0, matchProba, ldctx, &seed);
- }
+ /* Generate compressible data */
+ while (total < size)
+ {
+ RDG_genBlock(buff, RDG_DICTSIZE+RDG_BLOCKSIZE, RDG_DICTSIZE, matchProba, ldctx, &seed);
+ if (size-total < RDG_BLOCKSIZE) genBlockSize = (size_t)(size-total);
+ total += genBlockSize;
+ fwrite(buff, 1, genBlockSize, stdout);
+ /* update dict */
+ memcpy(buff, buff + RDG_BLOCKSIZE, RDG_DICTSIZE);
}
- // Get Seed
- DISPLAYLEVEL(4, "Data Generator %s \n", LZ4_VERSION);
- DISPLAYLEVEL(3, "Seed = %u \n", seed);
- if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", proba);
-
- CDG_generate(size, &seed, ((double)proba) / 100);
-
- return 0;
+ free(ldctx);
}
diff --git a/programs/datagen.h b/programs/datagen.h
new file mode 100644
index 0000000..631d146
--- /dev/null
+++ b/programs/datagen.h
@@ -0,0 +1,40 @@
+/*
+ datagen.h - compressible data generator header
+ Copyright (C) Yann Collet 2012-2015
+
+ GPL v2 License
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+ You can contact the author at :
+ - ZSTD source repository : https://github.com/Cyan4973/zstd
+ - Public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+
+
+#include <stddef.h> /* size_t */
+
+void RDG_genOut(unsigned long long size, double matchProba, double litProba, unsigned seed);
+void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed);
+/* RDG_genOut
+ Generate 'size' bytes of compressible data into stdout.
+ Compressibility can be controlled using 'matchProba'.
+ 'LitProba' is optional, and affect variability of bytes. If litProba==0.0, default value is used.
+ Generated data can be selected using 'seed'.
+ If (matchProba, litProba and seed) are equal, the function always generate the same content.
+
+ RDG_genBuffer
+ Same as RDG_genOut, but generate data into provided buffer
+*/
diff --git a/programs/datagencli.c b/programs/datagencli.c
new file mode 100644
index 0000000..601cb0a
--- /dev/null
+++ b/programs/datagencli.c
@@ -0,0 +1,193 @@
+/*
+ datagencli.c
+ compressible data command line generator
+ Copyright (C) Yann Collet 2012-2015
+
+ GPL v2 License
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+ You can contact the author at :
+ - ZSTD source repository : https://github.com/Cyan4973/zstd
+ - Public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+
+/**************************************
+* Includes
+**************************************/
+#include <stdio.h> /* fprintf, stderr */
+#include "datagen.h" /* RDG_generate */
+
+
+/**************************************
+* Basic Types
+**************************************/
+#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
+# include <stdint.h>
+ typedef uint8_t BYTE;
+ typedef uint16_t U16;
+ typedef uint32_t U32;
+ typedef int32_t S32;
+ typedef uint64_t U64;
+#else
+ typedef unsigned char BYTE;
+ typedef unsigned short U16;
+ typedef unsigned int U32;
+ typedef signed int S32;
+ typedef unsigned long long U64;
+#endif
+
+
+/**************************************
+* Constants
+**************************************/
+#ifndef ZSTD_VERSION
+# define ZSTD_VERSION "r1"
+#endif
+
+#define KB *(1 <<10)
+#define MB *(1 <<20)
+#define GB *(1U<<30)
+
+#define SIZE_DEFAULT (64 KB)
+#define SEED_DEFAULT 0
+#define COMPRESSIBILITY_DEFAULT 50
+
+
+/**************************************
+* Macros
+**************************************/
+#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+static unsigned displayLevel = 2;
+
+
+/*********************************************************
+* Command line
+*********************************************************/
+static int usage(char* programName)
+{
+ DISPLAY( "Compressible data generator\n");
+ DISPLAY( "Usage :\n");
+ DISPLAY( " %s [size] [args]\n", programName);
+ DISPLAY( "\n");
+ DISPLAY( "Arguments :\n");
+ DISPLAY( " -g# : generate # data (default:%i)\n", SIZE_DEFAULT);
+ DISPLAY( " -s# : Select seed (default:%i)\n", SEED_DEFAULT);
+ DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", COMPRESSIBILITY_DEFAULT);
+ DISPLAY( " -h : display help and exit\n");
+ DISPLAY( "Special values :\n");
+ DISPLAY( " -P0 : generate incompressible noise\n");
+ DISPLAY( " -P100 : generate sparse files\n");
+ return 0;
+}
+
+
+int main(int argc, char** argv)
+{
+ int argNb;
+ double proba = (double)COMPRESSIBILITY_DEFAULT / 100;
+ double litProba = 0.0;
+ U64 size = SIZE_DEFAULT;
+ U32 seed = SEED_DEFAULT;
+ char* programName;
+
+ /* Check command line */
+ programName = argv[0];
+ for(argNb=1; argNb<argc; argNb++)
+ {
+ char* argument = argv[argNb];
+
+ if(!argument) continue; /* Protection if argument empty */
+
+ /* Handle commands. Aggregated commands are allowed */
+ if (*argument=='-')
+ {
+ argument++;
+ while (*argument!=0)
+ {
+ switch(*argument)
+ {
+ case 'h':
+ return usage(programName);
+ case 'g':
+ argument++;
+ size=0;
+ while ((*argument>='0') && (*argument<='9'))
+ {
+ size *= 10;
+ size += *argument - '0';
+ argument++;
+ }
+ if (*argument=='K') { size <<= 10; argument++; }
+ if (*argument=='M') { size <<= 20; argument++; }
+ if (*argument=='G') { size <<= 30; argument++; }
+ if (*argument=='B') { argument++; }
+ break;
+ case 's':
+ argument++;
+ seed=0;
+ while ((*argument>='0') && (*argument<='9'))
+ {
+ seed *= 10;
+ seed += *argument - '0';
+ argument++;
+ }
+ break;
+ case 'P':
+ argument++;
+ proba=0.0;
+ while ((*argument>='0') && (*argument<='9'))
+ {
+ proba *= 10;
+ proba += *argument - '0';
+ argument++;
+ }
+ if (proba>100.) proba=100.;
+ proba /= 100.;
+ break;
+ case 'L': /* hidden argument : Literal distribution probability */
+ argument++;
+ litProba=0.;
+ while ((*argument>='0') && (*argument<='9'))
+ {
+ litProba *= 10;
+ litProba += *argument - '0';
+ argument++;
+ }
+ if (litProba>100.) litProba=100.;
+ litProba /= 100.;
+ break;
+ case 'v':
+ displayLevel = 4;
+ argument++;
+ break;
+ default:
+ return usage(programName);
+ }
+ }
+
+ }
+ }
+
+ DISPLAYLEVEL(4, "Data Generator %s \n", ZSTD_VERSION);
+ DISPLAYLEVEL(3, "Seed = %u \n", seed);
+ if (proba!=COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", (U32)(proba*100));
+
+ RDG_genOut(size, proba, litProba, seed);
+ DISPLAYLEVEL(1, "\n");
+
+ return 0;
+}
diff --git a/programs/frametest.c b/programs/frametest.c
index a5a23c2..df37e5f 100644
--- a/programs/frametest.c
+++ b/programs/frametest.c
@@ -229,11 +229,12 @@ int basicTests(U32 seed, double compressibility)
void* decodedBuffer;
U32 randState = seed;
size_t cSize, testSize;
- LZ4F_preferences_t prefs = { 0 };
+ LZ4F_preferences_t prefs;
LZ4F_decompressionContext_t dCtx;
U64 crcOrig;
// Create compressible test buffer
+ memset(&prefs, 0, sizeof(prefs));
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
compressedBuffer = malloc(LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL));
decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
@@ -441,7 +442,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
unsigned BMId = FUZ_rand(&randState) & 1;
unsigned CCflag = FUZ_rand(&randState) & 1;
unsigned autoflush = (FUZ_rand(&randState) & 7) == 2;
- LZ4F_preferences_t prefs = { 0 };
+ LZ4F_preferences_t prefs;
LZ4F_compressOptions_t cOptions = { 0 };
LZ4F_decompressOptions_t dOptions = { 0 };
unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(srcDataLength-1) - 1)) + 1;
@@ -452,6 +453,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
LZ4F_preferences_t* prefsPtr = &prefs;
(void)FUZ_rand(&coreRand); // update rand seed
+ memset(&prefs, 0, sizeof(prefs));
prefs.frameInfo.blockMode = (blockMode_t)BMId;
prefs.frameInfo.blockSizeID = (blockSizeID_t)BSId;
prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)CCflag;
@@ -523,17 +525,14 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
if (oSize > (size_t)(oend-op)) oSize = oend-op;
dOptions.stableDst = FUZ_rand(&randState) & 1;
if (nonContiguousDst==2) dOptions.stableDst = 0;
- //if (ip == compressedBuffer+62073) DISPLAY("oSize : %i : pos %i \n", (int)oSize, (int)(op-(BYTE*)decodedBuffer));
result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions);
- //if (op+oSize >= (BYTE*)decodedBuffer+94727) DISPLAY("iSize : %i : pos %i \n", (int)iSize, (int)(ip-(BYTE*)compressedBuffer));
- //if ((int)result<0) DISPLAY("iSize : %i : pos %i \n", (int)iSize, (int)(ip-(BYTE*)compressedBuffer));
if (result == (size_t)-ERROR_checksum_invalid) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst);
CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s)", (int)result, LZ4F_getErrorName((LZ4F_errorCode_t)result));
XXH64_update(&xxh64, op, (U32)oSize);
op += oSize;
ip += iSize;
op += nonContiguousDst;
- if (nonContiguousDst==2) op = decodedBuffer; // overwritten destination
+ if (nonContiguousDst==2) op = (BYTE*)decodedBuffer; /* overwritten destination */
}
CHECK(result != 0, "Frame decompression failed (error %i)", (int)result);
crcDecoded = XXH64_digest(&xxh64);
diff --git a/programs/fullbench.c b/programs/fullbench.c
index 756357a..6ae28d7 100755..100644
--- a/programs/fullbench.c
+++ b/programs/fullbench.c
@@ -687,7 +687,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
milliTime = BMK_GetMilliStart();
while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
{
- if (initFunction!=NULL) ctx = initFunction(chunkP[0].origBuffer);
+ if (initFunction!=NULL) ctx = (LZ4_stream_t*)initFunction(chunkP[0].origBuffer);
for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
{
chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
diff --git a/programs/fuzzer.c b/programs/fuzzer.c
index f9467d9..ee2beb8 100644
--- a/programs/fuzzer.c
+++ b/programs/fuzzer.c
@@ -323,6 +323,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
U32 crcOrig, crcCheck;
U32 coreRandState = seed;
U32 randState = coreRandState ^ PRIME3;
+ int result = 0;
// init
@@ -688,7 +689,6 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
// unalloc
{
- int result = 0;
_exit:
free(CNBuffer);
free(compressedBuffer);
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index ef3ef65..28b7fd6 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -19,7 +19,7 @@
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- - LZ4 source repository : http://code.google.com/p/lz4/
+ - LZ4 source repository : https://github.com/Cyan4973/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
/*
@@ -176,6 +176,7 @@ static int usage_advanced(void)
DISPLAY( " -BD : Block dependency (improve compression ratio)\n");
/* DISPLAY( " -BX : enable block checksum (default:disabled)\n"); *//* Option currently inactive */
DISPLAY( " -Sx : disable stream checksum (default:enabled)\n");
+ DISPLAY( " -X : enable sparse file (default:disabled)(experimental)\n");
DISPLAY( "Benchmark arguments :\n");
DISPLAY( " -b : benchmark file(s)\n");
DISPLAY( " -i# : iteration loops [1-9](default : 3), benchmark mode only\n");
@@ -375,7 +376,7 @@ int main(int argc, char** argv)
{
int B = argument[1] - '0';
blockSize = LZ4IO_setBlockSizeID(B);
- BMK_SetBlocksize(blockSize);
+ BMK_setBlocksize(blockSize);
argument++;
break;
}
@@ -390,16 +391,19 @@ int main(int argc, char** argv)
/* Modify Stream properties */
case 'S': if (argument[1]=='x') { LZ4IO_setStreamChecksumMode(0); argument++; break; } else { badusage(); }
+ /* Enable Sparse File support (experimental) */
+ case 'X': LZ4IO_setSparseFile(1); break;
+
/* Benchmark */
case 'b': bench=1; multiple_inputs=1;
if (inFileNames == NULL)
- inFileNames = malloc(argc * sizeof(char*));
+ inFileNames = (const char**) malloc(argc * sizeof(char*));
break;
/* Treat non-option args as input files. See https://code.google.com/p/lz4/issues/detail?id=151 */
case 'm': multiple_inputs=1;
if (inFileNames == NULL)
- inFileNames = malloc(argc * sizeof(char*));
+ inFileNames = (const char**) malloc(argc * sizeof(char*));
break;
/* Modify Nb Iterations (benchmark only) */
@@ -407,13 +411,13 @@ int main(int argc, char** argv)
if ((argument[1] >='1') && (argument[1] <='9'))
{
int iters = argument[1] - '0';
- BMK_SetNbIterations(iters);
+ BMK_setNbIterations(iters);
argument++;
}
break;
/* Pause at the end (hidden option) */
- case 'p': main_pause=1; BMK_SetPause(); break;
+ case 'p': main_pause=1; BMK_setPause(); break;
/* Specific commands for customized versions */
EXTENDED_ARGUMENTS;
@@ -454,7 +458,7 @@ int main(int argc, char** argv)
if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage();
/* Check if benchmark is selected */
- if (bench) return BMK_benchFile(inFileNames, ifnIdx, cLevel);
+ if (bench) return BMK_benchFiles(inFileNames, ifnIdx, cLevel);
/* No output filename ==> try to select one automatically (when possible) */
while (!output_filename)
diff --git a/programs/lz4io.c b/programs/lz4io.c
index bc8f3ff..94bf5ea 100644..100755
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -1,6 +1,7 @@
/*
LZ4io.c - LZ4 File/Stream Interface
Copyright (C) Yann Collet 2011-2015
+
GPL v2 License
This program is free software; you can redistribute it and/or modify
@@ -18,7 +19,7 @@
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- - LZ4 source repository : http://code.google.com/p/lz4/
+ - LZ4 source repository : https://github.com/Cyan4973/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
/*
@@ -31,28 +32,21 @@
/**************************************
* Compiler Options
-***************************************/
+**************************************/
#ifdef _MSC_VER /* Visual Studio */
# define _CRT_SECURE_NO_WARNINGS
# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#endif
-#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#ifdef __GNUC__
-# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
-# pragma GCC diagnostic ignored "-Wmissing-field-initializers" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
-#endif
-
#define _LARGE_FILES /* Large file support on 32-bits AIX */
#define _FILE_OFFSET_BITS 64 /* Large file support on 32-bits unix */
-#define _POSIX_SOURCE 1 /* for fileno() within <stdio.h> on unix */
-/****************************
+/*****************************
* Includes
*****************************/
-#include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */
+#include <stdio.h> /* fprintf, fopen, fread, stdin, stdout */
#include <stdlib.h> /* malloc, free */
#include <string.h> /* strcmp, strlen */
#include <time.h> /* clock */
@@ -62,28 +56,25 @@
#include "lz4frame.h"
-/****************************
+/******************************
* OS-specific Includes
-*****************************/
+******************************/
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
# include <fcntl.h> /* _O_BINARY */
-# include <io.h> /* _setmode, _isatty */
-# ifdef __MINGW32__
- int _fileno(FILE *stream); /* MINGW somehow forgets to include this windows declaration into <stdio.h> */
-# endif
+# include <io.h> /* _setmode, _fileno, _get_osfhandle */
+# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
+# include <Windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */
+# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); }
# if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */
# define fseek _fseeki64
# endif
-# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
-# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
#else
-# include <unistd.h> /* isatty */
# define SET_BINARY_MODE(file)
-# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
+# define SET_SPARSE_FILE_MODE(file)
#endif
-/****************************
+/*****************************
* Constants
*****************************/
#define KB *(1 <<10)
@@ -113,26 +104,28 @@
/**************************************
* Macros
-***************************************/
+**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
-#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
-#define DISPLAYUPDATE(l, ...) if (displayLevel>=l) { \
- if ((LZ4IO_GetMilliSpan(g_time) > refreshRate) || (displayLevel>=4)) \
+#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+static int g_displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */
+
+#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
+ if ((LZ4IO_GetMilliSpan(g_time) > refreshRate) || (g_displayLevel>=4)) \
{ g_time = clock(); DISPLAY(__VA_ARGS__); \
- if (displayLevel>=4) fflush(stdout); } }
+ if (g_displayLevel>=4) fflush(stdout); } }
static const unsigned refreshRate = 150;
static clock_t g_time = 0;
/**************************************
* Local Parameters
-***************************************/
-static int displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */
-static int overwrite = 1;
-static int globalBlockSizeId = LZ4S_BLOCKSIZEID_DEFAULT;
-static int blockChecksum = 0;
-static int streamChecksum = 1;
-static int blockIndependence = 1;
+**************************************/
+static int g_overwrite = 1;
+static int g_blockSizeId = LZ4S_BLOCKSIZEID_DEFAULT;
+static int g_blockChecksum = 0;
+static int g_streamChecksum = 1;
+static int g_blockIndependence = 1;
+static int g_sparseFileSupport = 0;
static const int minBlockSizeID = 4;
static const int maxBlockSizeID = 7;
@@ -159,7 +152,6 @@ static const int maxBlockSizeID = 7;
#define EXTENDED_ARGUMENTS
#define EXTENDED_HELP
#define EXTENDED_FORMAT
-#define DEFAULT_COMPRESSOR compress_file
#define DEFAULT_DECOMPRESSOR decodeLZ4S
@@ -170,8 +162,8 @@ static const int maxBlockSizeID = 7;
/* Default setting : overwrite = 1; return : overwrite mode (0/1) */
int LZ4IO_setOverwrite(int yes)
{
- overwrite = (yes!=0);
- return overwrite;
+ g_overwrite = (yes!=0);
+ return g_overwrite;
}
/* blockSizeID : valid values : 4-5-6-7 */
@@ -179,35 +171,42 @@ int LZ4IO_setBlockSizeID(int bsid)
{
static const int blockSizeTable[] = { 64 KB, 256 KB, 1 MB, 4 MB };
if ((bsid < minBlockSizeID) || (bsid > maxBlockSizeID)) return -1;
- globalBlockSizeId = bsid;
- return blockSizeTable[globalBlockSizeId-minBlockSizeID];
+ g_blockSizeId = bsid;
+ return blockSizeTable[g_blockSizeId-minBlockSizeID];
}
int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode)
{
- blockIndependence = (blockMode == LZ4IO_blockIndependent);
- return blockIndependence;
+ g_blockIndependence = (blockMode == LZ4IO_blockIndependent);
+ return g_blockIndependence;
}
/* Default setting : no checksum */
int LZ4IO_setBlockChecksumMode(int xxhash)
{
- blockChecksum = (xxhash != 0);
- return blockChecksum;
+ g_blockChecksum = (xxhash != 0);
+ return g_blockChecksum;
}
/* Default setting : checksum enabled */
int LZ4IO_setStreamChecksumMode(int xxhash)
{
- streamChecksum = (xxhash != 0);
- return streamChecksum;
+ g_streamChecksum = (xxhash != 0);
+ return g_streamChecksum;
}
/* Default setting : 0 (no notification) */
int LZ4IO_setNotificationLevel(int level)
{
- displayLevel = level;
- return displayLevel;
+ g_displayLevel = level;
+ return g_displayLevel;
+}
+
+/* Default setting : 0 (disabled) */
+int LZ4IO_setSparseFile(int enable)
+{
+ g_sparseFileSupport = enable;
+ return g_sparseFileSupport;
}
static unsigned LZ4IO_GetMilliSpan(clock_t nPrevious)
@@ -218,9 +217,9 @@ static unsigned LZ4IO_GetMilliSpan(clock_t nPrevious)
}
-/* ************************************************************************ */
-/* ********************** LZ4 File / Pipe compression ********************* */
-/* ************************************************************************ */
+/* ************************************************************************ **
+** ********************** LZ4 File / Pipe compression ********************* **
+** ************************************************************************ */
static int LZ4S_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); }
static int LZ4S_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4S_SKIPPABLEMASK) == LZ4S_SKIPPABLE0; }
@@ -254,12 +253,12 @@ static int get_fileHandle(const char* input_filename, const char* output_filenam
if (*pfoutput!=0)
{
fclose(*pfoutput);
- if (!overwrite)
+ if (!g_overwrite)
{
char ch;
DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename);
DISPLAYLEVEL(2, "Overwrite ? (Y/N) : ");
- if (displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); /* No interaction possible */
+ if (g_displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); /* No interaction possible */
ch = (char)getchar();
if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename);
}
@@ -277,13 +276,13 @@ static int get_fileHandle(const char* input_filename, const char* output_filenam
/***************************************
- * Legacy Compression
- * *************************************/
+* Legacy Compression
+***************************************/
/* unoptimized version; solves endianess & alignment issues */
static void LZ4IO_writeLE32 (void* p, unsigned value32)
{
- unsigned char* dstPtr = p;
+ unsigned char* dstPtr = (unsigned char*)p;
dstPtr[0] = (unsigned char)value32;
dstPtr[1] = (unsigned char)(value32 >> 8);
dstPtr[2] = (unsigned char)(value32 >> 16);
@@ -311,7 +310,7 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
if (compressionlevel < 3) compressionFunction = LZ4_compress; else compressionFunction = LZ4_compressHC;
get_fileHandle(input_filename, output_filename, &finput, &foutput);
- if ((displayLevel==2) && (compressionlevel==1)) displayLevel=3;
+ if ((g_displayLevel==2) && (compressionlevel==1)) g_displayLevel=3;
/* Allocate Memory */
in_buff = (char*)malloc(LEGACY_BLOCKSIZE);
@@ -363,9 +362,9 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
}
-/***********************************************
- * Compression using Frame format
- * ********************************************/
+/*********************************************
+* Compression using Frame format
+*********************************************/
int LZ4IO_compressFilename(const char* input_filename, const char* output_filename, int compressionLevel)
{
@@ -380,23 +379,24 @@ int LZ4IO_compressFilename(const char* input_filename, const char* output_filena
size_t sizeCheck, headerSize, readSize, outBuffSize;
LZ4F_compressionContext_t ctx;
LZ4F_errorCode_t errorCode;
- LZ4F_preferences_t prefs = {0};
+ LZ4F_preferences_t prefs;
/* Init */
start = clock();
- if ((displayLevel==2) && (compressionLevel>=3)) displayLevel=3;
+ memset(&prefs, 0, sizeof(prefs));
+ if ((g_displayLevel==2) && (compressionLevel>=3)) g_displayLevel=3;
errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode));
get_fileHandle(input_filename, output_filename, &finput, &foutput);
- blockSize = LZ4S_GetBlockSize_FromBlockId (globalBlockSizeId);
+ blockSize = LZ4S_GetBlockSize_FromBlockId (g_blockSizeId);
/* Set compression parameters */
prefs.autoFlush = 1;
prefs.compressionLevel = compressionLevel;
- prefs.frameInfo.blockMode = blockIndependence;
- prefs.frameInfo.blockSizeID = globalBlockSizeId;
- prefs.frameInfo.contentChecksumFlag = streamChecksum;
+ prefs.frameInfo.blockMode = (blockMode_t)g_blockIndependence;
+ prefs.frameInfo.blockSizeID = (blockSizeID_t)g_blockSizeId;
+ prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)g_streamChecksum;
/* Allocate Memory */
in_buff = (char*)malloc(blockSize);
@@ -475,7 +475,7 @@ int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize,
for (i=0; i<ifntSize; i++)
{
size_t ifnSize = strlen(inFileNamesTable[i]);
- if (ofnSize <= ifnSize+suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = malloc(ofnSize); }
+ if (ofnSize <= ifnSize+suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); }
strcpy(outFileName, inFileNamesTable[i]);
strcat(outFileName, suffix);
LZ4IO_compressFilename(inFileNamesTable[i], outFileName, compressionlevel);
@@ -492,7 +492,7 @@ int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize,
static unsigned LZ4IO_readLE32 (const void* s)
{
- const unsigned char* srcPtr = s;
+ const unsigned char* srcPtr = (const unsigned char*)s;
unsigned value32 = srcPtr[0];
value32 += (srcPtr[1]<<8);
value32 += (srcPtr[2]<<16);
@@ -552,14 +552,15 @@ static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput)
static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
{
unsigned long long filesize = 0;
- char* inBuff;
- char* outBuff;
+ void* inBuff;
+ void* outBuff;
# define HEADERMAX 20
char headerBuff[HEADERMAX];
size_t sizeCheck, nextToRead, outBuffSize, inBuffSize;
LZ4F_decompressionContext_t ctx;
LZ4F_errorCode_t errorCode;
LZ4F_frameInfo_t frameInfo;
+ unsigned storedSkips = 0;
/* init */
errorCode = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION);
@@ -580,8 +581,8 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
/* Allocate Memory */
outBuffSize = LZ4IO_setBlockSizeID(frameInfo.blockSizeID);
inBuffSize = outBuffSize + 4;
- inBuff = (char*)malloc(inBuffSize);
- outBuff = (char*)malloc(outBuffSize);
+ inBuff = malloc(inBuffSize);
+ outBuff = malloc(outBuffSize);
if (!inBuff || !outBuff) EXM_THROW(65, "Allocation error : not enough memory");
/* Main Loop */
@@ -601,8 +602,61 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
filesize += decodedBytes;
/* Write Block */
- sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput);
- if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block\n");
+ if (g_sparseFileSupport)
+ {
+ char* const oBuffStart = (char*)outBuff;
+ char* oBuffPos = oBuffStart;
+ char* const oBuffEnd = oBuffStart + decodedBytes;
+ static const size_t zeroBlockSize = 32 KB;
+ while (oBuffPos < oBuffEnd)
+ {
+ const size_t* sPtr = (const size_t*)oBuffPos;
+ size_t seg0Size = zeroBlockSize;
+ size_t nbSizeT;
+ size_t checked;
+ size_t skippedLength;
+ int seekResult;
+ if (seg0Size > decodedBytes) seg0Size = decodedBytes;
+ decodedBytes -= seg0Size;
+ nbSizeT = seg0Size / sizeof(size_t);
+ for (checked=0; (checked < nbSizeT) && (sPtr[checked] == 0); checked++) ;
+ skippedLength = checked * sizeof(size_t);
+ storedSkips += (unsigned)skippedLength;
+ if (storedSkips > 1 GB)
+ {
+ seekResult = fseek(foutput, 1 GB, SEEK_CUR);
+ if (seekResult != 0) EXM_THROW(68, "1 GB skip error (sparse file)\n");
+ storedSkips -= 1 GB;
+ }
+ if (skippedLength != seg0Size)
+ {
+ seekResult = fseek(foutput, storedSkips, SEEK_CUR);
+ if (seekResult != 0) EXM_THROW(68, "Skip error (sparse file)\n");
+ storedSkips = 0;
+ seg0Size -= skippedLength;
+ oBuffPos += skippedLength;
+ sizeCheck = fwrite(oBuffPos, 1, seg0Size, foutput);
+ if (sizeCheck != seg0Size) EXM_THROW(68, "Write error : cannot write decoded block\n");
+ }
+ oBuffPos += seg0Size;
+ }
+ }
+ else
+ {
+ sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput);
+ if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block\n");
+ }
+ }
+
+ if ((g_sparseFileSupport) && (storedSkips>0))
+ {
+ int seekResult;
+ storedSkips --;
+ seekResult = fseek(foutput, storedSkips, SEEK_CUR);
+ if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n");
+ memset(outBuff, 0, 1);
+ sizeCheck = fwrite(outBuff, 1, 1, foutput);
+ if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n");
}
/* Free */
@@ -666,6 +720,12 @@ int LZ4IO_decompressFilename(const char* input_filename, const char* output_file
start = clock();
get_fileHandle(input_filename, output_filename, &finput, &foutput);
+ /* sparse file */
+ if (g_sparseFileSupport && foutput)
+ {
+ SET_SPARSE_FILE_MODE(foutput);
+ }
+
/* Loop over multiple streams */
do
{
diff --git a/programs/lz4io.h b/programs/lz4io.h
index 2270c65..2441174 100644
--- a/programs/lz4io.h
+++ b/programs/lz4io.h
@@ -18,7 +18,7 @@
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- - LZ4 source repository : http://code.google.com/p/lz4/
+ - LZ4 source repository : https://github.com/Cyan4973/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
/*
@@ -29,6 +29,7 @@
- The license of this source file is GPLv2.
*/
+#pragma once
/* ************************************************** */
/* Special input/output values */
@@ -69,11 +70,15 @@ int LZ4IO_setBlockSizeID(int blockSizeID);
typedef enum { LZ4IO_blockLinked=0, LZ4IO_blockIndependent} LZ4IO_blockMode_t;
int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode);
-/* Default setting : no checksum */
+/* Default setting : no block checksum */
int LZ4IO_setBlockChecksumMode(int xxhash);
-/* Default setting : checksum enabled */
+/* Default setting : stream checksum enabled */
int LZ4IO_setStreamChecksumMode(int xxhash);
/* Default setting : 0 (no notification) */
int LZ4IO_setNotificationLevel(int level);
+
+/* Default setting : 0 (disabled) */
+int LZ4IO_setSparseFile(int enable);
+
generated by cgit v0.12 at 2024-07-15 22:56:39 (GMT)