diff options
author | Yann Collet <yann.collet.73@gmail.com> | 2015-03-15 00:34:05 (GMT) |
---|---|---|
committer | Yann Collet <yann.collet.73@gmail.com> | 2015-03-15 00:34:05 (GMT) |
commit | a18fb4392a9f249528a4071a5583e17668d92872 (patch) | |
tree | 56cf1740e2843b81960bbcdf3adc1c8a768f3583 | |
parent | 74a6b14f291c1041a6b36dfb2722268b703c0e96 (diff) | |
parent | 248b761f55b92a34eacbfe2ac5d6f1b94e69d7bf (diff) | |
download | lz4-a18fb4392a9f249528a4071a5583e17668d92872.zip lz4-a18fb4392a9f249528a4071a5583e17668d92872.tar.gz lz4-a18fb4392a9f249528a4071a5583e17668d92872.tar.bz2 |
Merge pull request #58 from Cyan4973/sparseFile
Sparse file
-rw-r--r-- | .travis.yml | 2 | ||||
-rw-r--r-- | Makefile | 8 | ||||
-rw-r--r-- | NEWS | 3 | ||||
-rw-r--r-- | lib/lz4.c | 15 | ||||
-rw-r--r-- | lib/lz4.h | 5 | ||||
-rw-r--r-- | lib/lz4frame.c | 126 | ||||
-rw-r--r-- | lib/lz4frame.h | 5 | ||||
-rw-r--r-- | lib/lz4frame_static.h | 3 | ||||
-rw-r--r-- | lib/lz4hc.c | 7 | ||||
-rw-r--r-- | lib/lz4hc.h | 6 | ||||
-rw-r--r-- | lib/xxhash.c | 3 | ||||
-rw-r--r-- | programs/Makefile | 24 | ||||
-rw-r--r-- | programs/bench.c | 15 | ||||
-rw-r--r-- | programs/bench.h | 21 | ||||
-rwxr-xr-x[-rw-r--r--] | programs/datagen.c | 329 | ||||
-rw-r--r-- | programs/datagen.h | 40 | ||||
-rw-r--r-- | programs/datagencli.c | 193 | ||||
-rw-r--r-- | programs/frametest.c | 11 | ||||
-rw-r--r--[-rwxr-xr-x] | programs/fullbench.c | 2 | ||||
-rw-r--r-- | programs/fuzzer.c | 2 | ||||
-rw-r--r-- | programs/lz4cli.c | 18 | ||||
-rwxr-xr-x[-rw-r--r--] | programs/lz4io.c | 206 | ||||
-rw-r--r-- | programs/lz4io.h | 11 |
23 files changed, 648 insertions, 407 deletions
diff --git a/.travis.yml b/.travis.yml index 9eff075..650fba1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ compiler: gcc script: make test-travis before_install: - sudo apt-get update -qq + - sudo apt-get install -qq g++-multilib - sudo apt-get install -qq gcc-multilib - sudo apt-get install -qq valgrind @@ -10,6 +11,7 @@ env: - LZ4_TRAVIS_CI_ENV=travis-install - LZ4_TRAVIS_CI_ENV=streaming-examples - LZ4_TRAVIS_CI_ENV=cmake + - LZ4_TRAVIS_CI_ENV=gpptest - LZ4_TRAVIS_CI_ENV=dist - LZ4_TRAVIS_CI_ENV=test-lz4 - LZ4_TRAVIS_CI_ENV=test-lz4c @@ -50,7 +50,7 @@ TEXT = $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4.h $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4hc.h \ Makefile lz4_block_format.txt LZ4_Frame_Format.html NEWS README.md \ cmake_unofficial/CMakeLists.txt \ $(PRGDIR)/fullbench.c $(PRGDIR)/lz4cli.c \ - $(PRGDIR)/datagen.c $(PRGDIR)/fuzzer.c \ + $(PRGDIR)/datagen.c $(PRGDIR)/datagen.h $(PRGDIR)/datagencli.c $(PRGDIR)/fuzzer.c \ $(PRGDIR)/lz4io.c $(PRGDIR)/lz4io.h \ $(PRGDIR)/bench.c $(PRGDIR)/bench.h \ $(PRGDIR)/lz4.1 $(PRGDIR)/lz4c.1 $(PRGDIR)/lz4cat.1 \ @@ -82,7 +82,7 @@ clean: @rm -f $(DISTRIBNAME) *.sha1 @cd $(PRGDIR); $(MAKE) clean @cd $(LZ4DIR); $(MAKE) clean - @cd examples; $(MAKE) clean + @cd examples; $(MAKE) clean @echo Cleaning completed @@ -127,6 +127,10 @@ test-travis: $(TRAVIS_TARGET) cmake: @cd cmake_unofficial; cmake CMakeLists.txt; $(MAKE) +gpptest: + export CC=g++; export CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align"; $(MAKE) -e all + + streaming-examples: cd examples; $(MAKE) -e test @@ -1,6 +1,9 @@ r128: +New : lz4 cli sparse file support New : command -m, to compress multiple files in a single command Fixed : Restored lz4hc compression ratio (slightly lower since r124) +New : g++ compatibility test +New : datagen can generate sparse files Fixed : Fuzzer + frametest compatibility with NetBSD (issue #48) Added : Visual project directory @@ -27,8 +27,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4 - - LZ4 source mirror : https://github.com/Cyan4973/lz4 + - LZ4 source repository : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ @@ -190,7 +189,7 @@ static U16 LZ4_readLE16(const void* memPtr) return *(U16*)memPtr; else { - const BYTE* p = memPtr; + const BYTE* p = (const BYTE*)memPtr; return (U16)((U16)p[0] + (p[1]<<8)); } } @@ -204,7 +203,7 @@ static void LZ4_writeLE16(void* memPtr, U16 value) } else { - BYTE* p = memPtr; + BYTE* p = (BYTE*)memPtr; p[0] = (BYTE) value; p[1] = (BYTE)(value>>8); } @@ -285,9 +284,9 @@ static void LZ4_copy8(void* dstPtr, const void* srcPtr) /* customized version of memcpy, which may overwrite up to 7 bytes beyond dstEnd */ static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) { - BYTE* d = dstPtr; - const BYTE* s = srcPtr; - BYTE* e = dstEnd; + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* e = (BYTE*)dstEnd; do { LZ4_copy8(d,s); d+=8; s+=8; } while (d<e); } @@ -1151,7 +1150,7 @@ typedef struct */ LZ4_streamDecode_t* LZ4_createStreamDecode(void) { - LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(sizeof(U64), LZ4_STREAMDECODESIZE_U64); + LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(LZ4_STREAMDECODESIZE_U64, sizeof(U64)); return lz4s; } @@ -1,7 +1,8 @@ /* LZ4 - Fast LZ compression algorithm Header File - Copyright (C) 2011-2014, Yann Collet. + Copyright (C) 2011-2015, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -28,7 +29,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 source repository : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ #pragma once diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 5183f22..7153871 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -1,6 +1,7 @@ /* LZ4 auto-framing library -Copyright (C) 2011-2014, Yann Collet. +Copyright (C) 2011-2015, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -27,13 +28,13 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : -- LZ4 source repository : http://code.google.com/p/lz4/ +- LZ4 source repository : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ /* LZ4F is a stand-alone API to create LZ4-compressed Frames -* fully conformant to specification v1.4.1. -* All related operations, including memory management, are handled by the library. +* in full conformance with specification v1.4.1. +* All related operations, including memory management, are handled by the library. * */ @@ -45,14 +46,10 @@ Compiler Options #endif #define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#ifdef __GNUC__ -# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ -# pragma GCC diagnostic ignored "-Wmissing-field-initializers" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ -#endif /************************************** -Memory routines +* Memory routines **************************************/ #include <stdlib.h> /* malloc, calloc, free */ #define ALLOCATOR(s) calloc(1,s) @@ -62,7 +59,7 @@ Memory routines /************************************** -Includes +* Includes **************************************/ #include "lz4frame_static.h" #include "lz4.h" @@ -71,7 +68,7 @@ Includes /************************************** -Basic Types +* Basic Types **************************************/ #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ # include <stdint.h> @@ -90,7 +87,7 @@ typedef unsigned long long U64; /************************************** -Constants +* Constants **************************************/ #define KB *(1<<10) #define MB *(1<<20) @@ -110,7 +107,7 @@ Constants static const U32 minHClevel = 3; /************************************** -Structures and local types +* Structures and local types **************************************/ typedef struct { @@ -150,12 +147,7 @@ typedef struct /************************************** -Macros -**************************************/ - - -/************************************** -Error management +* Error management **************************************/ #define LZ4F_GENERATE_STRING(STRING) #STRING, static const char* LZ4F_errorStrings[] = { LZ4F_LIST_ERRORS(LZ4F_GENERATE_STRING) }; @@ -175,7 +167,7 @@ const char* LZ4F_getErrorName(LZ4F_errorCode_t code) /************************************** -Private functions +* Private functions **************************************/ static size_t LZ4F_getBlockSize(unsigned blockSizeID) { @@ -215,14 +207,15 @@ static BYTE LZ4F_headerChecksum (const BYTE* header, size_t length) /************************************** -Simple compression functions +* Simple compression functions **************************************/ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { - LZ4F_preferences_t prefs = { 0 }; + LZ4F_preferences_t prefs; size_t headerSize; size_t streamSize; + memset(&prefs, 0, sizeof(prefs)); if (preferencesPtr!=NULL) prefs = *preferencesPtr; { blockSizeID_t proposedBSID = max64KB; @@ -234,7 +227,7 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere prefs.frameInfo.blockSizeID = proposedBSID; break; } - proposedBSID++; + proposedBSID = (blockSizeID_t)( ((int)proposedBSID) + 1); maxBlockSize <<= 2; } } @@ -258,14 +251,17 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere */ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { - LZ4F_cctx_internal_t cctxI = { 0 }; /* works because no allocation */ - LZ4F_preferences_t prefs = { 0 }; - LZ4F_compressOptions_t options = { 0 }; + LZ4F_cctx_internal_t cctxI; + LZ4F_preferences_t prefs; + LZ4F_compressOptions_t options; LZ4F_errorCode_t errorCode; BYTE* const dstStart = (BYTE*) dstBuffer; BYTE* dstPtr = dstStart; BYTE* const dstEnd = dstStart + dstMaxSize; + memset(&cctxI, 0, sizeof(cctxI)); /* works because no allocation */ + memset(&prefs, 0, sizeof(prefs)); + memset(&options, 0, sizeof(options)); cctxI.version = LZ4F_VERSION; cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent allocation; works because autoflush==1 & stableSrc==1 */ @@ -281,7 +277,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf prefs.frameInfo.blockSizeID = proposedBSID; break; } - proposedBSID++; + proposedBSID = (blockSizeID_t)((int)proposedBSID + 1); maxBlockSize <<= 2; } } @@ -361,7 +357,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp */ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_preferences_t* preferencesPtr) { - LZ4F_preferences_t prefNull = { 0 }; + LZ4F_preferences_t prefNull; LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; @@ -370,6 +366,7 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds if (dstMaxSize < LZ4F_MAXHEADERFRAME_SIZE) return (size_t)-ERROR_dstMaxSize_tooSmall; if (cctxPtr->cStage != 0) return (size_t)-ERROR_GENERIC; + memset(&prefNull, 0, sizeof(prefNull)); if (preferencesPtr == NULL) preferencesPtr = &prefNull; cctxPtr->prefs = *preferencesPtr; @@ -436,17 +433,19 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds * */ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { - const LZ4F_preferences_t prefsNull = { 0 }; - const LZ4F_preferences_t* prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; - blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID; - size_t blockSize = LZ4F_getBlockSize(bid); - unsigned nbBlocks = (unsigned)(srcSize / blockSize) + 1; - size_t lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize; - size_t blockInfo = 4; /* default, without block CRC option */ - size_t frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); - size_t result = (blockInfo * nbBlocks) + (blockSize * (nbBlocks-1)) + lastBlockSize + frameEnd; - - return result; + LZ4F_preferences_t prefsNull; + memset(&prefsNull, 0, sizeof(prefsNull)); + { + const LZ4F_preferences_t* prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; + blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID; + size_t blockSize = LZ4F_getBlockSize(bid); + unsigned nbBlocks = (unsigned)(srcSize / blockSize) + 1; + size_t lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize; + size_t blockInfo = 4; /* default, without block CRC option */ + size_t frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); + + return (blockInfo * nbBlocks) + (blockSize * (nbBlocks-1)) + lastBlockSize + frameEnd;; + } } @@ -518,7 +517,7 @@ typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus; */ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr) { - LZ4F_compressOptions_t cOptionsNull = { 0 }; + LZ4F_compressOptions_t cOptionsNull; LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; size_t blockSize = cctxPtr->maxBlockSize; const BYTE* srcPtr = (const BYTE*)srcBuffer; @@ -531,6 +530,7 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d if (cctxPtr->cStage != 1) return (size_t)-ERROR_GENERIC; if (dstMaxSize < LZ4F_compressBound(srcSize, &(cctxPtr->prefs))) return (size_t)-ERROR_dstMaxSize_tooSmall; + memset(&cOptionsNull, 0, sizeof(cOptionsNull)); if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; /* select compression function */ @@ -627,7 +627,7 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d */ size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) { - LZ4F_compressOptions_t cOptionsNull = { 0 }; + LZ4F_compressOptions_t cOptionsNull; LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; @@ -637,8 +637,8 @@ size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */ if (cctxPtr->cStage != 1) return (size_t)-ERROR_GENERIC; if (dstMaxSize < (cctxPtr->tmpInSize + 16)) return (size_t)-ERROR_dstMaxSize_tooSmall; + memset(&cOptionsNull, 0, sizeof(cOptionsNull)); if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; - (void)compressOptionsPtr; /* not yet useful */ /* select compression function */ compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); @@ -712,7 +712,7 @@ LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_compressionContext_t* LZ4F { LZ4F_dctx_internal_t* dctxPtr; - dctxPtr = ALLOCATOR(sizeof(LZ4F_dctx_internal_t)); + dctxPtr = (LZ4F_dctx_internal_t*)ALLOCATOR(sizeof(LZ4F_dctx_internal_t)); if (dctxPtr==NULL) return (LZ4F_errorCode_t)-ERROR_GENERIC; dctxPtr->version = versionNumber; @@ -732,11 +732,12 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_compressionContext_t LZ4F_de /* Decompression */ -static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPtr, size_t srcSize) +static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const void* srcVoidPtr, size_t srcSize) { BYTE FLG, BD, HC; unsigned version, blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, dictFlag, blockSizeID; size_t bufferNeeded; + const BYTE* srcPtr = (const BYTE*)srcVoidPtr; /* need to decode header to get frameInfo */ if (srcSize < 7) return (size_t)-ERROR_GENERIC; /* minimal header size */ @@ -747,7 +748,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPt /* Flags */ FLG = srcPtr[0]; - version = (FLG>>6)&_2BITS; + version = (FLG>>6) & _2BITS; blockMode = (FLG>>5) & _1BIT; blockChecksumFlag = (FLG>>4) & _1BIT; contentSizeFlag = (FLG>>3) & _1BIT; @@ -761,19 +762,19 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPt if (HC != srcPtr[2]) return (size_t)-ERROR_GENERIC; /* Bad header checksum error */ /* validate */ - if (version != 1) return (size_t)-ERROR_GENERIC; /* Version Number, only supported value */ - if (blockChecksumFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */ + if (version != 1) return (size_t)-ERROR_GENERIC; /* Version Number, only supported value */ + if (blockChecksumFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */ if (contentSizeFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */ - if (((FLG>>1)&_1BIT) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bit */ - if (dictFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */ + if (((FLG>>1)&_1BIT) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bit */ + if (dictFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */ if (((BD>>7)&_1BIT) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bit */ - if (blockSizeID < 4) return (size_t)-ERROR_GENERIC; /* Only supported values for the time being */ - if (((BD>>0)&_4BITS) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bits */ + if (blockSizeID < 4) return (size_t)-ERROR_GENERIC; /* 4-7 only supported values for the time being */ + if (((BD>>0)&_4BITS) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bits */ /* save */ - dctxPtr->frameInfo.blockMode = blockMode; - dctxPtr->frameInfo.contentChecksumFlag = contentChecksumFlag; - dctxPtr->frameInfo.blockSizeID = blockSizeID; + dctxPtr->frameInfo.blockMode = (blockMode_t)blockMode; + dctxPtr->frameInfo.contentChecksumFlag = (contentChecksum_t)contentChecksumFlag; + dctxPtr->frameInfo.blockSizeID = (blockSizeID_t)blockSizeID; dctxPtr->maxBlockSize = LZ4F_getBlockSize(blockSizeID); /* init */ @@ -786,9 +787,9 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPt FREEMEM(dctxPtr->tmpIn); FREEMEM(dctxPtr->tmpOutBuffer); dctxPtr->maxBufferSize = bufferNeeded; - dctxPtr->tmpIn = ALLOCATOR(dctxPtr->maxBlockSize); + dctxPtr->tmpIn = (BYTE*)ALLOCATOR(dctxPtr->maxBlockSize); if (dctxPtr->tmpIn == NULL) return (size_t)-ERROR_GENERIC; - dctxPtr->tmpOutBuffer= ALLOCATOR(dctxPtr->maxBufferSize); + dctxPtr->tmpOutBuffer= (BYTE*)ALLOCATOR(dctxPtr->maxBufferSize); if (dctxPtr->tmpOutBuffer== NULL) return (size_t)-ERROR_GENERIC; } dctxPtr->tmpInSize = 0; @@ -880,15 +881,6 @@ static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, s if (withinTmp) /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */ { -#if 0 - size_t savedDictSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; - memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart- savedDictSize, savedDictSize); - dctxPtr->dict = dctxPtr->tmpOutBuffer; - dctxPtr->dictSize = savedDictSize + dctxPtr->tmpOutStart + dstSize; - return; - -#else - size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; size_t copySize = 64 KB - dctxPtr->tmpOutSize; BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; @@ -900,7 +892,6 @@ static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, s dctxPtr->dict = dctxPtr->tmpOutBuffer; dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart + dstSize; return; -#endif } if (dctxPtr->dict == dctxPtr->tmpOutBuffer) /* copy dst into tmp to complete dict */ @@ -952,7 +943,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, const LZ4F_decompressOptions_t* decompressOptionsPtr) { LZ4F_dctx_internal_t* dctxPtr = (LZ4F_dctx_internal_t*)decompressionContext; - static const LZ4F_decompressOptions_t optionsNull = { 0 }; + LZ4F_decompressOptions_t optionsNull; const BYTE* const srcStart = (const BYTE*)srcBuffer; const BYTE* const srcEnd = srcStart + *srcSizePtr; const BYTE* srcPtr = srcStart; @@ -964,6 +955,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, size_t nextSrcSizeHint = 1; + memset(&optionsNull, 0, sizeof(optionsNull)); if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull; *srcSizePtr = 0; *dstSizePtr = 0; diff --git a/lib/lz4frame.h b/lib/lz4frame.h index d73e3e2..61e461b 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -28,8 +28,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ - - LZ4 source mirror : https://github.com/Cyan4973/lz4 + - LZ4 source repository : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ @@ -215,7 +214,7 @@ size_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t ctx, * The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). * You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr) * The function result is an hint of how many srcSize bytes LZ4F_decompress() expects for next call, - * or an error code which can be tested using LZ4F_isError(). + * or an error code which can be tested using LZ4F_isError(). */ size_t LZ4F_decompress(LZ4F_decompressionContext_t ctx, diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h index cde8186..4c34c6c 100644 --- a/lib/lz4frame_static.h +++ b/lib/lz4frame_static.h @@ -29,8 +29,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ - - LZ4 source mirror : https://github.com/Cyan4973/lz4 + - LZ4 source repository : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 5549969..357fa96 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -1,6 +1,7 @@ /* LZ4 HC - High Compression Mode of LZ4 -Copyright (C) 2011-2014, Yann Collet. +Copyright (C) 2011-2015, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -27,8 +28,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : -- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html -- LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 source repository : https://github.com/Cyan4973/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ diff --git a/lib/lz4hc.h b/lib/lz4hc.h index ce813ab..eb72051 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -1,7 +1,7 @@ /* LZ4 HC - High Compression Mode of LZ4 Header File - Copyright (C) 2011-2014, Yann Collet. + Copyright (C) 2011-2015, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -28,8 +28,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 source repository : https://github.com/Cyan4973/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ #pragma once diff --git a/lib/xxhash.c b/lib/xxhash.c index 093564c..aca1e0a 100644 --- a/lib/xxhash.c +++ b/lib/xxhash.c @@ -28,8 +28,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : -- xxHash source repository : http://code.google.com/p/xxhash/ -- xxHash source mirror : https://github.com/Cyan4973/xxHash +- xxHash source repository : https://github.com/Cyan4973/xxHash - public discussion board : https://groups.google.com/forum/#!forum/lz4c */ diff --git a/programs/Makefile b/programs/Makefile index 1070f40..3d38244 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -92,7 +92,7 @@ frametest: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxha frametest32: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c frametest.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) -datagen : datagen.c +datagen : datagen.c datagencli.c $(CC) $(FLAGS) $^ -o $@$(EXT) @@ -142,11 +142,11 @@ test-travis: $(TRAVIS_TARGET) test-lz4: lz4 datagen ./datagen -g16KB | ./lz4 -9 | ./lz4 -vdq > $(VOID) ./datagen | ./lz4 | ./lz4 -vdq > $(VOID) - ./datagen -g6M -p100 | ./lz4 -9BD | ./lz4 -vdq > $(VOID) - ./datagen -g17M | ./lz4 -9v | ./lz4 -vdq > $(VOID) + ./datagen -g6M -P100 | ./lz4 -9BD | ./lz4 -vdq > $(VOID) + ./datagen -g17M | ./lz4 -9v | ./lz4 -dq > $(VOID) ./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -vdq > $(VOID) ./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -vdq > $(VOID) -# test frame concatenation with null-length frame + @echo ---- test frame concatenation ---- @echo -n > empty.test @echo hi > nonempty.test cat nonempty.test empty.test nonempty.test > orig.test @@ -157,13 +157,24 @@ test-lz4: lz4 datagen sdiff orig.test result.test @rm *.test @echo frame concatenation test completed -# test frame concatenation with null-length frame - @echo test multiple input files + @echo ---- test multiple input files ---- @./datagen -s1 > file1 @./datagen -s2 > file2 @./datagen -s3 > file3 ./lz4 -f -m file1 file2 file3 + ls -l file* @rm file1 file2 file3 file1.lz4 file2.lz4 file3.lz4 + @echo ---- test sparse file support ---- + ./datagen -g50M -P100 | ./lz4 -B4D | ./lz4 -dvX > tmpB4 + ./datagen -g50M -P100 | ./lz4 -B5D | ./lz4 -dvX > tmpB5 + ./datagen -g50M -P100 | ./lz4 -B6D | ./lz4 -dvX > tmpB6 + ./datagen -g50M -P100 | ./lz4 -B7D | ./lz4 -dvX > tmpB7 + ls -ls tmp* + ./datagen -g50M -P100 | diff -s - tmpB4 + ./datagen -g50M -P100 | diff -s - tmpB5 + ./datagen -g50M -P100 | diff -s - tmpB6 + ./datagen -g50M -P100 | diff -s - tmpB7 + @rm tmp* test-lz4c: lz4c datagen @@ -197,6 +208,7 @@ test-frametest32: frametest32 ./frametest32 test-mem: lz4 datagen fuzzer frametest + valgrind --leak-check=yes ./datagen -g50M > /dev/null ./datagen -g16KB > tmp valgrind --leak-check=yes ./lz4 -9 -BD -f tmp /dev/null ./datagen -g16MB > tmp diff --git a/programs/bench.c b/programs/bench.c index 77120f2..b632314 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -1,6 +1,7 @@ /* - bench.c - Demo program to benchmark open-source compression algorithm + bench.c - Demo program to benchmark open-source compression algorithms Copyright (C) Yann Collet 2012-2015 + GPL v2 License This program is free software; you can redistribute it and/or modify @@ -18,8 +19,8 @@ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. You can contact the author at : - - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 source repository : https://github.com/Cyan4973/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ /************************************** @@ -141,15 +142,15 @@ static int chunkSize = DEFAULT_CHUNKSIZE; static int nbIterations = NBLOOPS; static int BMK_pause = 0; -void BMK_SetBlocksize(int bsize) { chunkSize = bsize; } +void BMK_setBlocksize(int bsize) { chunkSize = bsize; } -void BMK_SetNbIterations(int nbLoops) +void BMK_setNbIterations(int nbLoops) { nbIterations = nbLoops; DISPLAY("- %i iterations -\n", nbIterations); } -void BMK_SetPause(void) { BMK_pause = 1; } +void BMK_setPause(void) { BMK_pause = 1; } /********************************************************* @@ -234,7 +235,7 @@ static U64 BMK_GetFileSize(const char* infilename) * Public function **********************************************************/ -int BMK_benchFile(const char** fileNamesTable, int nbFiles, int cLevel) +int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel) { int fileIdx=0; char* orig_buff; diff --git a/programs/bench.h b/programs/bench.h index 2a20cdb..c04fb17 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -17,26 +17,17 @@ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ - - LZ4 public forum : https://group.google.com/forum/#!forum/lz4c + - LZ4 source repository : https://github.com/Cyan4973/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ #pragma once -#if defined (__cplusplus) -extern "C" { -#endif - /* Main function */ -int BMK_benchFile(const char** fileNamesTable, int nbFiles, int cLevel); +int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel); /* Set Parameters */ -void BMK_SetBlocksize(int bsize); -void BMK_SetNbIterations(int nbLoops); -void BMK_SetPause(void); - - +void BMK_setBlocksize(int bsize); +void BMK_setNbIterations(int nbLoops); +void BMK_setPause(void); -#if defined (__cplusplus) -} -#endif diff --git a/programs/datagen.c b/programs/datagen.c index 0f07477..bccb21e 100644..100755 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -19,26 +19,20 @@ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4 - - LZ4 source mirror : https://github.com/Cyan4973/lz4 - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c + - ZSTD source repository : https://github.com/Cyan4973/zstd + - Public forum : https://groups.google.com/forum/#!forum/lz4c */ /************************************** - Remove Visual warning messages +* Includes **************************************/ -#define _CRT_SECURE_NO_WARNINGS // fgets +#include <stdlib.h> /* malloc */ +#include <stdio.h> /* FILE, fwrite */ +#include <string.h> /* memcpy */ /************************************** - Includes -**************************************/ -#include <stdio.h> // fgets, sscanf -#include <string.h> // strcmp - - -/************************************** - Basic Types +* Basic Types **************************************/ #if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ # include <stdint.h> @@ -57,230 +51,173 @@ /************************************** - Constants +* OS-specific Includes **************************************/ -#ifndef LZ4_VERSION -# define LZ4_VERSION "r125" +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +# include <fcntl.h> /* _O_BINARY */ +# include <io.h> /* _setmode, _isatty */ +# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) +#else +# define SET_BINARY_MODE(file) #endif -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -#define CDG_SIZE_DEFAULT (64 KB) -#define CDG_SEED_DEFAULT 0 -#define CDG_COMPRESSIBILITY_DEFAULT 50 -#define PRIME1 2654435761U -#define PRIME2 2246822519U - /************************************** - Macros +* Constants **************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } - +#define KB *(1 <<10) -/************************************** - Local Parameters -**************************************/ -static unsigned no_prompt = 0; -static char* programName; -static unsigned displayLevel = 2; +#define PRIME1 2654435761U +#define PRIME2 2246822519U /********************************************************* - functions +* Local Functions *********************************************************/ - -#define CDG_rotl32(x,r) ((x << r) | (x >> (32 - r))) -static unsigned int CDG_rand(U32* src) +#define RDG_rotl32(x,r) ((x << r) | (x >> (32 - r))) +static unsigned int RDG_rand(U32* src) { U32 rand32 = *src; rand32 *= PRIME1; - rand32 += PRIME2; - rand32 = CDG_rotl32(rand32, 13); + rand32 ^= PRIME2; + rand32 = RDG_rotl32(rand32, 13); *src = rand32; return rand32; } -#define CDG_RAND15BITS ((CDG_rand(seed) >> 3) & 32767) -#define CDG_RANDLENGTH ( ((CDG_rand(seed) >> 7) & 3) ? (CDG_rand(seed) % 14) : (CDG_rand(seed) & 511) + 15) -#define CDG_RANDCHAR (((CDG_rand(seed) >> 9) & 63) + '0') -static void CDG_generate(U64 size, U32* seed, double proba) +#define LTSIZE 8192 +#define LTMASK (LTSIZE-1) +static void* RDG_createLiteralDistrib(double ld) { - BYTE fullbuff[32 KB + 128 KB + 1]; - BYTE* buff = fullbuff + 32 KB; - U64 total=0; - U32 P32 = (U32)(32768 * proba); - U32 pos=1; - U32 genBlockSize = 128 KB; + BYTE* lt = (BYTE*)malloc(LTSIZE); + U32 i = 0; + BYTE character = '0'; + BYTE firstChar = '('; + BYTE lastChar = '}'; - // Build initial prefix - fullbuff[0] = CDG_RANDCHAR; - while (pos<32 KB) + if (ld==0.0) { - // Select : Literal (char) or Match (within 32K) - if (CDG_RAND15BITS < P32) - { - // Copy (within 64K) - U32 d; - int ref; - int length = CDG_RANDLENGTH + 4; - U32 offset = CDG_RAND15BITS + 1; - if (offset > pos) offset = pos; - ref = pos - offset; - d = pos + length; - while (pos < d) fullbuff[pos++] = fullbuff[ref++]; - } - else + character = 0; + firstChar = 0; + lastChar =255; + } + while (i<LTSIZE) + { + U32 weight = (U32)((double)(LTSIZE - i) * ld) + 1; + U32 end; + if (weight + i > LTSIZE) weight = LTSIZE-i; + end = i + weight; + while (i < end) lt[i++] = character; + character++; + if (character > lastChar) character = firstChar; + } + return lt; +} + +static char RDG_genChar(U32* seed, const void* ltctx) +{ + const BYTE* lt = (const BYTE*)ltctx; + U32 id = RDG_rand(seed) & LTMASK; + return lt[id]; +} + +#define RDG_DICTSIZE (32 KB) +#define RDG_RAND15BITS ((RDG_rand(seed) >> 3) & 32767) +#define RDG_RANDLENGTH ( ((RDG_rand(seed) >> 7) & 7) ? (RDG_rand(seed) & 15) : (RDG_rand(seed) & 511) + 15) +void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, void* litTable, unsigned* seedPtr) +{ + BYTE* buffPtr = (BYTE*)buffer; + const U32 matchProba32 = (U32)(32768 * matchProba); + size_t pos = prefixSize; + void* ldctx = litTable; + U32* seed = seedPtr; + + /* special case */ + while (matchProba >= 1.0) + { + size_t size0 = RDG_rand(seed) & 3; + size0 = (size_t)1 << (16 + size0 * 2); + size0 += RDG_rand(seed) & (size0-1); /* because size0 is power of 2*/ + if (buffSize < pos + size0) { - // Literal (noise) - U32 d = pos + CDG_RANDLENGTH; - while (pos < d) fullbuff[pos++] = CDG_RANDCHAR; + memset(buffPtr+pos, 0, buffSize-pos); + return; } + memset(buffPtr+pos, 0, size0); + pos += size0; + buffPtr[pos-1] = RDG_genChar(seed, ldctx); } - // Generate compressible data - pos = 0; - while (total < size) + /* init */ + if (pos==0) buffPtr[0] = RDG_genChar(seed, ldctx), pos=1; + + /* Generate compressible data */ + while (pos < buffSize) { - if (size-total < 128 KB) genBlockSize = (U32)(size-total); - total += genBlockSize; - buff[genBlockSize] = 0; - pos = 0; - while (pos<genBlockSize) + /* Select : Literal (char) or Match (within 32K) */ + if (RDG_RAND15BITS < matchProba32) + { + /* Copy (within 32K) */ + size_t match; + size_t d; + int length = RDG_RANDLENGTH + 4; + U32 offset = RDG_RAND15BITS + 1; + if (offset > pos) offset = (U32)pos; + match = pos - offset; + d = pos + length; + if (d > buffSize) d = buffSize; + while (pos < d) buffPtr[pos++] = buffPtr[match++]; + } + else { - // Select : Literal (char) or Match (within 32K) - if (CDG_RAND15BITS < P32) - { - // Copy (within 64K) - int ref; - U32 d; - int length = CDG_RANDLENGTH + 4; - U32 offset = CDG_RAND15BITS + 1; - if (pos + length > genBlockSize ) length = genBlockSize - pos; - ref = pos - offset; - d = pos + length; - while (pos < d) buff[pos++] = buff[ref++]; - } - else - { - // Literal (noise) - U32 d; - int length = CDG_RANDLENGTH; - if (pos + length > genBlockSize) length = genBlockSize - pos; - d = pos + length; - while (pos < d) buff[pos++] = CDG_RANDCHAR; - } + /* Literal (noise) */ + size_t d; + size_t length = RDG_RANDLENGTH; + d = pos + length; + if (d > buffSize) d = buffSize; + while (pos < d) buffPtr[pos++] = RDG_genChar(seed, ldctx); } - // output datagen - pos=0; - for (;pos+512<=genBlockSize;pos+=512) - printf("%512.512s", buff+pos); - for (;pos<genBlockSize;pos++) printf("%c", buff[pos]); - // Regenerate prefix - memcpy(fullbuff, buff + 96 KB, 32 KB); } } -int CDG_usage(void) +void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed) { - DISPLAY( "Compressible data generator\n"); - DISPLAY( "Usage :\n"); - DISPLAY( " %s [size] [args]\n", programName); - DISPLAY( "\n"); - DISPLAY( "Arguments :\n"); - DISPLAY( " -g# : generate # data (default:%i)\n", CDG_SIZE_DEFAULT); - DISPLAY( " -s# : Select seed (default:%i)\n", CDG_SEED_DEFAULT); - DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", CDG_COMPRESSIBILITY_DEFAULT); - DISPLAY( " -h : display help and exit\n"); - return 0; + void* ldctx; + if (litProba==0.0) litProba = matchProba / 4.5; + ldctx = RDG_createLiteralDistrib(litProba); + RDG_genBlock(buffer, size, 0, matchProba, ldctx, &seed); + free(ldctx); } -int main(int argc, char** argv) +#define RDG_BLOCKSIZE (128 KB) +void RDG_genOut(unsigned long long size, double matchProba, double litProba, unsigned seed) { - int argNb; - int proba = CDG_COMPRESSIBILITY_DEFAULT; - U64 size = CDG_SIZE_DEFAULT; - U32 seed = CDG_SEED_DEFAULT; - - // Check command line - programName = argv[0]; - for(argNb=1; argNb<argc; argNb++) - { - char* argument = argv[argNb]; - - if(!argument) continue; // Protection if argument empty + BYTE buff[RDG_DICTSIZE + RDG_BLOCKSIZE]; + U64 total = 0; + size_t genBlockSize = RDG_BLOCKSIZE; + void* ldctx; - // Decode command (note : aggregated commands are allowed) - if (*argument=='-') - { - if (!strcmp(argument, "--no-prompt")) { no_prompt=1; continue; } + /* init */ + if (litProba==0.0) litProba = matchProba / 4.5; + ldctx = RDG_createLiteralDistrib(litProba); + SET_BINARY_MODE(stdout); - argument++; - while (*argument!=0) - { - switch(*argument) - { - case 'h': - return CDG_usage(); - case 'g': - argument++; - size=0; - while ((*argument>='0') && (*argument<='9')) - { - size *= 10; - size += *argument - '0'; - argument++; - } - if (*argument=='K') { size <<= 10; argument++; } - if (*argument=='M') { size <<= 20; argument++; } - if (*argument=='G') { size <<= 30; argument++; } - if (*argument=='B') { argument++; } - break; - case 's': - argument++; - seed=0; - while ((*argument>='0') && (*argument<='9')) - { - seed *= 10; - seed += *argument - '0'; - argument++; - } - break; - case 'p': - argument++; - proba=0; - while ((*argument>='0') && (*argument<='9')) - { - proba *= 10; - proba += *argument - '0'; - argument++; - } - if (proba<0) proba=0; - if (proba>100) proba=100; - break; - case 'v': - displayLevel = 4; - argument++; - break; - default: ; - } - } + /* Generate dict */ + RDG_genBlock(buff, RDG_DICTSIZE, 0, matchProba, ldctx, &seed); - } + /* Generate compressible data */ + while (total < size) + { + RDG_genBlock(buff, RDG_DICTSIZE+RDG_BLOCKSIZE, RDG_DICTSIZE, matchProba, ldctx, &seed); + if (size-total < RDG_BLOCKSIZE) genBlockSize = (size_t)(size-total); + total += genBlockSize; + fwrite(buff, 1, genBlockSize, stdout); + /* update dict */ + memcpy(buff, buff + RDG_BLOCKSIZE, RDG_DICTSIZE); } - // Get Seed - DISPLAYLEVEL(4, "Data Generator %s \n", LZ4_VERSION); - DISPLAYLEVEL(3, "Seed = %u \n", seed); - if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", proba); - - CDG_generate(size, &seed, ((double)proba) / 100); - - return 0; + free(ldctx); } diff --git a/programs/datagen.h b/programs/datagen.h new file mode 100644 index 0000000..631d146 --- /dev/null +++ b/programs/datagen.h @@ -0,0 +1,40 @@ +/* + datagen.h - compressible data generator header + Copyright (C) Yann Collet 2012-2015 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - ZSTD source repository : https://github.com/Cyan4973/zstd + - Public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + + +#include <stddef.h> /* size_t */ + +void RDG_genOut(unsigned long long size, double matchProba, double litProba, unsigned seed); +void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed); +/* RDG_genOut + Generate 'size' bytes of compressible data into stdout. + Compressibility can be controlled using 'matchProba'. + 'LitProba' is optional, and affect variability of bytes. If litProba==0.0, default value is used. + Generated data can be selected using 'seed'. + If (matchProba, litProba and seed) are equal, the function always generate the same content. + + RDG_genBuffer + Same as RDG_genOut, but generate data into provided buffer +*/ diff --git a/programs/datagencli.c b/programs/datagencli.c new file mode 100644 index 0000000..601cb0a --- /dev/null +++ b/programs/datagencli.c @@ -0,0 +1,193 @@ +/* + datagencli.c + compressible data command line generator + Copyright (C) Yann Collet 2012-2015 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - ZSTD source repository : https://github.com/Cyan4973/zstd + - Public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +/************************************** +* Includes +**************************************/ +#include <stdio.h> /* fprintf, stderr */ +#include "datagen.h" /* RDG_generate */ + + +/************************************** +* Basic Types +**************************************/ +#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ +# include <stdint.h> + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif + + +/************************************** +* Constants +**************************************/ +#ifndef ZSTD_VERSION +# define ZSTD_VERSION "r1" +#endif + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define SIZE_DEFAULT (64 KB) +#define SEED_DEFAULT 0 +#define COMPRESSIBILITY_DEFAULT 50 + + +/************************************** +* Macros +**************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static unsigned displayLevel = 2; + + +/********************************************************* +* Command line +*********************************************************/ +static int usage(char* programName) +{ + DISPLAY( "Compressible data generator\n"); + DISPLAY( "Usage :\n"); + DISPLAY( " %s [size] [args]\n", programName); + DISPLAY( "\n"); + DISPLAY( "Arguments :\n"); + DISPLAY( " -g# : generate # data (default:%i)\n", SIZE_DEFAULT); + DISPLAY( " -s# : Select seed (default:%i)\n", SEED_DEFAULT); + DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", COMPRESSIBILITY_DEFAULT); + DISPLAY( " -h : display help and exit\n"); + DISPLAY( "Special values :\n"); + DISPLAY( " -P0 : generate incompressible noise\n"); + DISPLAY( " -P100 : generate sparse files\n"); + return 0; +} + + +int main(int argc, char** argv) +{ + int argNb; + double proba = (double)COMPRESSIBILITY_DEFAULT / 100; + double litProba = 0.0; + U64 size = SIZE_DEFAULT; + U32 seed = SEED_DEFAULT; + char* programName; + + /* Check command line */ + programName = argv[0]; + for(argNb=1; argNb<argc; argNb++) + { + char* argument = argv[argNb]; + + if(!argument) continue; /* Protection if argument empty */ + + /* Handle commands. Aggregated commands are allowed */ + if (*argument=='-') + { + argument++; + while (*argument!=0) + { + switch(*argument) + { + case 'h': + return usage(programName); + case 'g': + argument++; + size=0; + while ((*argument>='0') && (*argument<='9')) + { + size *= 10; + size += *argument - '0'; + argument++; + } + if (*argument=='K') { size <<= 10; argument++; } + if (*argument=='M') { size <<= 20; argument++; } + if (*argument=='G') { size <<= 30; argument++; } + if (*argument=='B') { argument++; } + break; + case 's': + argument++; + seed=0; + while ((*argument>='0') && (*argument<='9')) + { + seed *= 10; + seed += *argument - '0'; + argument++; + } + break; + case 'P': + argument++; + proba=0.0; + while ((*argument>='0') && (*argument<='9')) + { + proba *= 10; + proba += *argument - '0'; + argument++; + } + if (proba>100.) proba=100.; + proba /= 100.; + break; + case 'L': /* hidden argument : Literal distribution probability */ + argument++; + litProba=0.; + while ((*argument>='0') && (*argument<='9')) + { + litProba *= 10; + litProba += *argument - '0'; + argument++; + } + if (litProba>100.) litProba=100.; + litProba /= 100.; + break; + case 'v': + displayLevel = 4; + argument++; + break; + default: + return usage(programName); + } + } + + } + } + + DISPLAYLEVEL(4, "Data Generator %s \n", ZSTD_VERSION); + DISPLAYLEVEL(3, "Seed = %u \n", seed); + if (proba!=COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", (U32)(proba*100)); + + RDG_genOut(size, proba, litProba, seed); + DISPLAYLEVEL(1, "\n"); + + return 0; +} diff --git a/programs/frametest.c b/programs/frametest.c index a5a23c2..df37e5f 100644 --- a/programs/frametest.c +++ b/programs/frametest.c @@ -229,11 +229,12 @@ int basicTests(U32 seed, double compressibility) void* decodedBuffer; U32 randState = seed; size_t cSize, testSize; - LZ4F_preferences_t prefs = { 0 }; + LZ4F_preferences_t prefs; LZ4F_decompressionContext_t dCtx; U64 crcOrig; // Create compressible test buffer + memset(&prefs, 0, sizeof(prefs)); CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); compressedBuffer = malloc(LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL)); decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); @@ -441,7 +442,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi unsigned BMId = FUZ_rand(&randState) & 1; unsigned CCflag = FUZ_rand(&randState) & 1; unsigned autoflush = (FUZ_rand(&randState) & 7) == 2; - LZ4F_preferences_t prefs = { 0 }; + LZ4F_preferences_t prefs; LZ4F_compressOptions_t cOptions = { 0 }; LZ4F_decompressOptions_t dOptions = { 0 }; unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(srcDataLength-1) - 1)) + 1; @@ -452,6 +453,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi LZ4F_preferences_t* prefsPtr = &prefs; (void)FUZ_rand(&coreRand); // update rand seed + memset(&prefs, 0, sizeof(prefs)); prefs.frameInfo.blockMode = (blockMode_t)BMId; prefs.frameInfo.blockSizeID = (blockSizeID_t)BSId; prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)CCflag; @@ -523,17 +525,14 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi if (oSize > (size_t)(oend-op)) oSize = oend-op; dOptions.stableDst = FUZ_rand(&randState) & 1; if (nonContiguousDst==2) dOptions.stableDst = 0; - //if (ip == compressedBuffer+62073) DISPLAY("oSize : %i : pos %i \n", (int)oSize, (int)(op-(BYTE*)decodedBuffer)); result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions); - //if (op+oSize >= (BYTE*)decodedBuffer+94727) DISPLAY("iSize : %i : pos %i \n", (int)iSize, (int)(ip-(BYTE*)compressedBuffer)); - //if ((int)result<0) DISPLAY("iSize : %i : pos %i \n", (int)iSize, (int)(ip-(BYTE*)compressedBuffer)); if (result == (size_t)-ERROR_checksum_invalid) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst); CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s)", (int)result, LZ4F_getErrorName((LZ4F_errorCode_t)result)); XXH64_update(&xxh64, op, (U32)oSize); op += oSize; ip += iSize; op += nonContiguousDst; - if (nonContiguousDst==2) op = decodedBuffer; // overwritten destination + if (nonContiguousDst==2) op = (BYTE*)decodedBuffer; /* overwritten destination */ } CHECK(result != 0, "Frame decompression failed (error %i)", (int)result); crcDecoded = XXH64_digest(&xxh64); diff --git a/programs/fullbench.c b/programs/fullbench.c index 756357a..6ae28d7 100755..100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -687,7 +687,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) milliTime = BMK_GetMilliStart(); while(BMK_GetMilliSpan(milliTime) < TIMELOOP) { - if (initFunction!=NULL) ctx = initFunction(chunkP[0].origBuffer); + if (initFunction!=NULL) ctx = (LZ4_stream_t*)initFunction(chunkP[0].origBuffer); for (chunkNb=0; chunkNb<nbChunks; chunkNb++) { chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize); diff --git a/programs/fuzzer.c b/programs/fuzzer.c index f9467d9..ee2beb8 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -323,6 +323,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do U32 crcOrig, crcCheck; U32 coreRandState = seed; U32 randState = coreRandState ^ PRIME3; + int result = 0; // init @@ -688,7 +689,6 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do // unalloc { - int result = 0; _exit: free(CNBuffer); free(compressedBuffer); diff --git a/programs/lz4cli.c b/programs/lz4cli.c index ef3ef65..28b7fd6 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -19,7 +19,7 @@ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 source repository : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ /* @@ -176,6 +176,7 @@ static int usage_advanced(void) DISPLAY( " -BD : Block dependency (improve compression ratio)\n"); /* DISPLAY( " -BX : enable block checksum (default:disabled)\n"); *//* Option currently inactive */ DISPLAY( " -Sx : disable stream checksum (default:enabled)\n"); + DISPLAY( " -X : enable sparse file (default:disabled)(experimental)\n"); DISPLAY( "Benchmark arguments :\n"); DISPLAY( " -b : benchmark file(s)\n"); DISPLAY( " -i# : iteration loops [1-9](default : 3), benchmark mode only\n"); @@ -375,7 +376,7 @@ int main(int argc, char** argv) { int B = argument[1] - '0'; blockSize = LZ4IO_setBlockSizeID(B); - BMK_SetBlocksize(blockSize); + BMK_setBlocksize(blockSize); argument++; break; } @@ -390,16 +391,19 @@ int main(int argc, char** argv) /* Modify Stream properties */ case 'S': if (argument[1]=='x') { LZ4IO_setStreamChecksumMode(0); argument++; break; } else { badusage(); } + /* Enable Sparse File support (experimental) */ + case 'X': LZ4IO_setSparseFile(1); break; + /* Benchmark */ case 'b': bench=1; multiple_inputs=1; if (inFileNames == NULL) - inFileNames = malloc(argc * sizeof(char*)); + inFileNames = (const char**) malloc(argc * sizeof(char*)); break; /* Treat non-option args as input files. See https://code.google.com/p/lz4/issues/detail?id=151 */ case 'm': multiple_inputs=1; if (inFileNames == NULL) - inFileNames = malloc(argc * sizeof(char*)); + inFileNames = (const char**) malloc(argc * sizeof(char*)); break; /* Modify Nb Iterations (benchmark only) */ @@ -407,13 +411,13 @@ int main(int argc, char** argv) if ((argument[1] >='1') && (argument[1] <='9')) { int iters = argument[1] - '0'; - BMK_SetNbIterations(iters); + BMK_setNbIterations(iters); argument++; } break; /* Pause at the end (hidden option) */ - case 'p': main_pause=1; BMK_SetPause(); break; + case 'p': main_pause=1; BMK_setPause(); break; /* Specific commands for customized versions */ EXTENDED_ARGUMENTS; @@ -454,7 +458,7 @@ int main(int argc, char** argv) if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage(); /* Check if benchmark is selected */ - if (bench) return BMK_benchFile(inFileNames, ifnIdx, cLevel); + if (bench) return BMK_benchFiles(inFileNames, ifnIdx, cLevel); /* No output filename ==> try to select one automatically (when possible) */ while (!output_filename) diff --git a/programs/lz4io.c b/programs/lz4io.c index bc8f3ff..94bf5ea 100644..100755 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -1,6 +1,7 @@ /* LZ4io.c - LZ4 File/Stream Interface Copyright (C) Yann Collet 2011-2015 + GPL v2 License This program is free software; you can redistribute it and/or modify @@ -18,7 +19,7 @@ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 source repository : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ /* @@ -31,28 +32,21 @@ /************************************** * Compiler Options -***************************************/ +**************************************/ #ifdef _MSC_VER /* Visual Studio */ # define _CRT_SECURE_NO_WARNINGS # define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ #endif -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#ifdef __GNUC__ -# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ -# pragma GCC diagnostic ignored "-Wmissing-field-initializers" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ -#endif - #define _LARGE_FILES /* Large file support on 32-bits AIX */ #define _FILE_OFFSET_BITS 64 /* Large file support on 32-bits unix */ -#define _POSIX_SOURCE 1 /* for fileno() within <stdio.h> on unix */ -/**************************** +/***************************** * Includes *****************************/ -#include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */ +#include <stdio.h> /* fprintf, fopen, fread, stdin, stdout */ #include <stdlib.h> /* malloc, free */ #include <string.h> /* strcmp, strlen */ #include <time.h> /* clock */ @@ -62,28 +56,25 @@ #include "lz4frame.h" -/**************************** +/****************************** * OS-specific Includes -*****************************/ +******************************/ #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) # include <fcntl.h> /* _O_BINARY */ -# include <io.h> /* _setmode, _isatty */ -# ifdef __MINGW32__ - int _fileno(FILE *stream); /* MINGW somehow forgets to include this windows declaration into <stdio.h> */ -# endif +# include <io.h> /* _setmode, _fileno, _get_osfhandle */ +# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) +# include <Windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */
+# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } # if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ # define fseek _fseeki64 # endif -# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) -# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) #else -# include <unistd.h> /* isatty */ # define SET_BINARY_MODE(file) -# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +# define SET_SPARSE_FILE_MODE(file) #endif -/**************************** +/***************************** * Constants *****************************/ #define KB *(1 <<10) @@ -113,26 +104,28 @@ /************************************** * Macros -***************************************/ +**************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } -#define DISPLAYUPDATE(l, ...) if (displayLevel>=l) { \ - if ((LZ4IO_GetMilliSpan(g_time) > refreshRate) || (displayLevel>=4)) \ +#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static int g_displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */ + +#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ + if ((LZ4IO_GetMilliSpan(g_time) > refreshRate) || (g_displayLevel>=4)) \ { g_time = clock(); DISPLAY(__VA_ARGS__); \ - if (displayLevel>=4) fflush(stdout); } } + if (g_displayLevel>=4) fflush(stdout); } } static const unsigned refreshRate = 150; static clock_t g_time = 0; /************************************** * Local Parameters -***************************************/ -static int displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */ -static int overwrite = 1; -static int globalBlockSizeId = LZ4S_BLOCKSIZEID_DEFAULT; -static int blockChecksum = 0; -static int streamChecksum = 1; -static int blockIndependence = 1; +**************************************/ +static int g_overwrite = 1; +static int g_blockSizeId = LZ4S_BLOCKSIZEID_DEFAULT; +static int g_blockChecksum = 0; +static int g_streamChecksum = 1; +static int g_blockIndependence = 1; +static int g_sparseFileSupport = 0; static const int minBlockSizeID = 4; static const int maxBlockSizeID = 7; @@ -159,7 +152,6 @@ static const int maxBlockSizeID = 7; #define EXTENDED_ARGUMENTS #define EXTENDED_HELP #define EXTENDED_FORMAT -#define DEFAULT_COMPRESSOR compress_file #define DEFAULT_DECOMPRESSOR decodeLZ4S @@ -170,8 +162,8 @@ static const int maxBlockSizeID = 7; /* Default setting : overwrite = 1; return : overwrite mode (0/1) */ int LZ4IO_setOverwrite(int yes) { - overwrite = (yes!=0); - return overwrite; + g_overwrite = (yes!=0); + return g_overwrite; } /* blockSizeID : valid values : 4-5-6-7 */ @@ -179,35 +171,42 @@ int LZ4IO_setBlockSizeID(int bsid) { static const int blockSizeTable[] = { 64 KB, 256 KB, 1 MB, 4 MB }; if ((bsid < minBlockSizeID) || (bsid > maxBlockSizeID)) return -1; - globalBlockSizeId = bsid; - return blockSizeTable[globalBlockSizeId-minBlockSizeID]; + g_blockSizeId = bsid; + return blockSizeTable[g_blockSizeId-minBlockSizeID]; } int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode) { - blockIndependence = (blockMode == LZ4IO_blockIndependent); - return blockIndependence; + g_blockIndependence = (blockMode == LZ4IO_blockIndependent); + return g_blockIndependence; } /* Default setting : no checksum */ int LZ4IO_setBlockChecksumMode(int xxhash) { - blockChecksum = (xxhash != 0); - return blockChecksum; + g_blockChecksum = (xxhash != 0); + return g_blockChecksum; } /* Default setting : checksum enabled */ int LZ4IO_setStreamChecksumMode(int xxhash) { - streamChecksum = (xxhash != 0); - return streamChecksum; + g_streamChecksum = (xxhash != 0); + return g_streamChecksum; } /* Default setting : 0 (no notification) */ int LZ4IO_setNotificationLevel(int level) { - displayLevel = level; - return displayLevel; + g_displayLevel = level; + return g_displayLevel; +} + +/* Default setting : 0 (disabled) */ +int LZ4IO_setSparseFile(int enable) +{ + g_sparseFileSupport = enable; + return g_sparseFileSupport; } static unsigned LZ4IO_GetMilliSpan(clock_t nPrevious) @@ -218,9 +217,9 @@ static unsigned LZ4IO_GetMilliSpan(clock_t nPrevious) } -/* ************************************************************************ */ -/* ********************** LZ4 File / Pipe compression ********************* */ -/* ************************************************************************ */ +/* ************************************************************************ ** +** ********************** LZ4 File / Pipe compression ********************* ** +** ************************************************************************ */ static int LZ4S_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } static int LZ4S_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4S_SKIPPABLEMASK) == LZ4S_SKIPPABLE0; } @@ -254,12 +253,12 @@ static int get_fileHandle(const char* input_filename, const char* output_filenam if (*pfoutput!=0) { fclose(*pfoutput); - if (!overwrite) + if (!g_overwrite) { char ch; DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename); DISPLAYLEVEL(2, "Overwrite ? (Y/N) : "); - if (displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); /* No interaction possible */ + if (g_displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); /* No interaction possible */ ch = (char)getchar(); if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); } @@ -277,13 +276,13 @@ static int get_fileHandle(const char* input_filename, const char* output_filenam /*************************************** - * Legacy Compression - * *************************************/ +* Legacy Compression +***************************************/ /* unoptimized version; solves endianess & alignment issues */ static void LZ4IO_writeLE32 (void* p, unsigned value32) { - unsigned char* dstPtr = p; + unsigned char* dstPtr = (unsigned char*)p; dstPtr[0] = (unsigned char)value32; dstPtr[1] = (unsigned char)(value32 >> 8); dstPtr[2] = (unsigned char)(value32 >> 16); @@ -311,7 +310,7 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output if (compressionlevel < 3) compressionFunction = LZ4_compress; else compressionFunction = LZ4_compressHC; get_fileHandle(input_filename, output_filename, &finput, &foutput); - if ((displayLevel==2) && (compressionlevel==1)) displayLevel=3; + if ((g_displayLevel==2) && (compressionlevel==1)) g_displayLevel=3; /* Allocate Memory */ in_buff = (char*)malloc(LEGACY_BLOCKSIZE); @@ -363,9 +362,9 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output } -/*********************************************** - * Compression using Frame format - * ********************************************/ +/********************************************* +* Compression using Frame format +*********************************************/ int LZ4IO_compressFilename(const char* input_filename, const char* output_filename, int compressionLevel) { @@ -380,23 +379,24 @@ int LZ4IO_compressFilename(const char* input_filename, const char* output_filena size_t sizeCheck, headerSize, readSize, outBuffSize; LZ4F_compressionContext_t ctx; LZ4F_errorCode_t errorCode; - LZ4F_preferences_t prefs = {0}; + LZ4F_preferences_t prefs; /* Init */ start = clock(); - if ((displayLevel==2) && (compressionLevel>=3)) displayLevel=3; + memset(&prefs, 0, sizeof(prefs)); + if ((g_displayLevel==2) && (compressionLevel>=3)) g_displayLevel=3; errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode)); get_fileHandle(input_filename, output_filename, &finput, &foutput); - blockSize = LZ4S_GetBlockSize_FromBlockId (globalBlockSizeId); + blockSize = LZ4S_GetBlockSize_FromBlockId (g_blockSizeId); /* Set compression parameters */ prefs.autoFlush = 1; prefs.compressionLevel = compressionLevel; - prefs.frameInfo.blockMode = blockIndependence; - prefs.frameInfo.blockSizeID = globalBlockSizeId; - prefs.frameInfo.contentChecksumFlag = streamChecksum; + prefs.frameInfo.blockMode = (blockMode_t)g_blockIndependence; + prefs.frameInfo.blockSizeID = (blockSizeID_t)g_blockSizeId; + prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)g_streamChecksum; /* Allocate Memory */ in_buff = (char*)malloc(blockSize); @@ -475,7 +475,7 @@ int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, for (i=0; i<ifntSize; i++) { size_t ifnSize = strlen(inFileNamesTable[i]); - if (ofnSize <= ifnSize+suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = malloc(ofnSize); } + if (ofnSize <= ifnSize+suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); } strcpy(outFileName, inFileNamesTable[i]); strcat(outFileName, suffix); LZ4IO_compressFilename(inFileNamesTable[i], outFileName, compressionlevel); @@ -492,7 +492,7 @@ int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, static unsigned LZ4IO_readLE32 (const void* s) { - const unsigned char* srcPtr = s; + const unsigned char* srcPtr = (const unsigned char*)s; unsigned value32 = srcPtr[0]; value32 += (srcPtr[1]<<8); value32 += (srcPtr[2]<<16); @@ -552,14 +552,15 @@ static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput) static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) { unsigned long long filesize = 0; - char* inBuff; - char* outBuff; + void* inBuff; + void* outBuff; # define HEADERMAX 20 char headerBuff[HEADERMAX]; size_t sizeCheck, nextToRead, outBuffSize, inBuffSize; LZ4F_decompressionContext_t ctx; LZ4F_errorCode_t errorCode; LZ4F_frameInfo_t frameInfo; + unsigned storedSkips = 0; /* init */ errorCode = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION); @@ -580,8 +581,8 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) /* Allocate Memory */ outBuffSize = LZ4IO_setBlockSizeID(frameInfo.blockSizeID); inBuffSize = outBuffSize + 4; - inBuff = (char*)malloc(inBuffSize); - outBuff = (char*)malloc(outBuffSize); + inBuff = malloc(inBuffSize); + outBuff = malloc(outBuffSize); if (!inBuff || !outBuff) EXM_THROW(65, "Allocation error : not enough memory"); /* Main Loop */ @@ -601,8 +602,61 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) filesize += decodedBytes; /* Write Block */ - sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput); - if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block\n"); + if (g_sparseFileSupport) + { + char* const oBuffStart = (char*)outBuff; + char* oBuffPos = oBuffStart; + char* const oBuffEnd = oBuffStart + decodedBytes; + static const size_t zeroBlockSize = 32 KB; + while (oBuffPos < oBuffEnd) + { + const size_t* sPtr = (const size_t*)oBuffPos; + size_t seg0Size = zeroBlockSize; + size_t nbSizeT; + size_t checked; + size_t skippedLength; + int seekResult; + if (seg0Size > decodedBytes) seg0Size = decodedBytes; + decodedBytes -= seg0Size; + nbSizeT = seg0Size / sizeof(size_t); + for (checked=0; (checked < nbSizeT) && (sPtr[checked] == 0); checked++) ; + skippedLength = checked * sizeof(size_t); + storedSkips += (unsigned)skippedLength; + if (storedSkips > 1 GB) + { + seekResult = fseek(foutput, 1 GB, SEEK_CUR); + if (seekResult != 0) EXM_THROW(68, "1 GB skip error (sparse file)\n"); + storedSkips -= 1 GB; + } + if (skippedLength != seg0Size) + { + seekResult = fseek(foutput, storedSkips, SEEK_CUR); + if (seekResult != 0) EXM_THROW(68, "Skip error (sparse file)\n"); + storedSkips = 0; + seg0Size -= skippedLength; + oBuffPos += skippedLength; + sizeCheck = fwrite(oBuffPos, 1, seg0Size, foutput); + if (sizeCheck != seg0Size) EXM_THROW(68, "Write error : cannot write decoded block\n"); + } + oBuffPos += seg0Size; + } + } + else + { + sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput); + if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block\n"); + } + } + + if ((g_sparseFileSupport) && (storedSkips>0)) + { + int seekResult; + storedSkips --; + seekResult = fseek(foutput, storedSkips, SEEK_CUR); + if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n"); + memset(outBuff, 0, 1); + sizeCheck = fwrite(outBuff, 1, 1, foutput); + if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n"); } /* Free */ @@ -666,6 +720,12 @@ int LZ4IO_decompressFilename(const char* input_filename, const char* output_file start = clock(); get_fileHandle(input_filename, output_filename, &finput, &foutput); + /* sparse file */ + if (g_sparseFileSupport && foutput)
+ {
+ SET_SPARSE_FILE_MODE(foutput);
+ } + /* Loop over multiple streams */ do { diff --git a/programs/lz4io.h b/programs/lz4io.h index 2270c65..2441174 100644 --- a/programs/lz4io.h +++ b/programs/lz4io.h @@ -18,7 +18,7 @@ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 source repository : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ /* @@ -29,6 +29,7 @@ - The license of this source file is GPLv2. */ +#pragma once /* ************************************************** */ /* Special input/output values */ @@ -69,11 +70,15 @@ int LZ4IO_setBlockSizeID(int blockSizeID); typedef enum { LZ4IO_blockLinked=0, LZ4IO_blockIndependent} LZ4IO_blockMode_t; int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode); -/* Default setting : no checksum */ +/* Default setting : no block checksum */ int LZ4IO_setBlockChecksumMode(int xxhash); -/* Default setting : checksum enabled */ +/* Default setting : stream checksum enabled */ int LZ4IO_setStreamChecksumMode(int xxhash); /* Default setting : 0 (no notification) */ int LZ4IO_setNotificationLevel(int level); + +/* Default setting : 0 (disabled) */ +int LZ4IO_setSparseFile(int enable); + |