From 97679fa5a17d79f87c0a2d1e2d44c3215d48958b Mon Sep 17 00:00:00 2001 From: Takayuki MATSUOKA Date: Thu, 12 Feb 2015 15:46:02 +0900 Subject: Google Code Issue 155: lz4 cli should support sparse file https://code.google.com/p/lz4/issues/detail?id=155 This is experimental implementation. Just a proof of concept. It works Linux and Windows. # Build To build experimental version, define 'LZ4IO_ENABLE_SPARSE_FILE' like the following command : make lz4programs 'CFLAGS=-O3 -DLZ4IO_ENABLE_SPARSE_FILE=1' ./programs/lz4 -h You will see "EXPERIMENTAL_SPARSE_FILE" as lz4 revision : "*** LZ4 command line interface 64-bits EXPERIMENTAL_SPARSE_FILE, by Yann Collet (...) ***" # Experiment This experimental version adds option "-x" for sparse file for decompression. You can use this option like this : ./programs/lz4 -9 -f my-file ./programs/lz4 -d -f -x my-file.lz4 my-file.lz4.out cmp my-file my-file.lz4.out --- programs/lz4cli.c | 17 +++++++++++++++ programs/lz4io.c | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ programs/lz4io.h | 5 +++++ 3 files changed, 87 insertions(+) diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 0da5dce..5b54143 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -83,6 +83,11 @@ /***************************** * Constants ******************************/ +#if defined(LZ4IO_ENABLE_SPARSE_FILE) +# undef LZ4_VERSION +# define LZ4_VERSION "EXPERIMENTAL_SPARSE_FILE" +#endif + #define COMPRESSOR_NAME "LZ4 command line interface" #ifndef LZ4_VERSION # define LZ4_VERSION "r126" @@ -186,6 +191,10 @@ static int usage_advanced(void) DISPLAY( " -y : overwrite output without prompting \n"); DISPLAY( " -s : suppress warnings \n"); #endif /* ENABLE_LZ4C_LEGACY_OPTIONS */ +#if defined(LZ4IO_ENABLE_SPARSE_FILE) + DISPLAY( "Experimental : Sparse file\n"); + DISPLAY( " -x : enable sparse file\n"); +#endif /* LZ4IO_ENABLE_SPARSE_FILE */ EXTENDED_HELP; return 0; } @@ -276,6 +285,9 @@ int main(int argc, char** argv) /* Init */ programName = argv[0]; LZ4IO_setOverwrite(0); +#if defined(LZ4IO_ENABLE_SPARSE_FILE) + LZ4IO_setSparseFile(0); +#endif /* LZ4IO_ENABLE_SPARSE_FILE */ blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT); /* lz4cat behavior */ @@ -403,6 +415,11 @@ int main(int argc, char** argv) /* Pause at the end (hidden option) */ case 'p': main_pause=1; BMK_SetPause(); break; +#if defined(LZ4IO_ENABLE_SPARSE_FILE) + /* Experimental : Enable sparse file */ + case 'x': LZ4IO_setSparseFile(1); break; +#endif /* LZ4IO_ENABLE_SPARSE_FILE */ + /* Specific commands for customized versions */ EXTENDED_ARGUMENTS; diff --git a/programs/lz4io.c b/programs/lz4io.c index 023824e..4e583f6 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -76,10 +76,18 @@ # endif # define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +# if defined(LZ4IO_ENABLE_SPARSE_FILE) +# include +# define SET_SPARSE_FILE_MODE(file) do { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } while(0) +# define fseek _fseeki64 +# endif /* LZ4IO_ENABLE_SPARSE_FILE */ #else # include /* isatty */ # define SET_BINARY_MODE(file) # define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +# if defined(LZ4IO_ENABLE_SPARSE_FILE) +# define SET_SPARSE_FILE_MODE(file) +# endif /* LZ4IO_ENABLE_SPARSE_FILE */ #endif @@ -133,6 +141,9 @@ static int globalBlockSizeId = LZ4S_BLOCKSIZEID_DEFAULT; static int blockChecksum = 0; static int streamChecksum = 1; static int blockIndependence = 1; +#if defined(LZ4IO_ENABLE_SPARSE_FILE) +static int sparseFile = 0; +#endif /* LZ4IO_ENABLE_SPARSE_FILE */ static const int minBlockSizeID = 4; static const int maxBlockSizeID = 7; @@ -174,6 +185,28 @@ int LZ4IO_setOverwrite(int yes) return overwrite; } +#if defined(LZ4IO_ENABLE_SPARSE_FILE) +/* Default setting : sparseFile = 0; (Disable) + return : sparse file mode (0:Disable / 1:Enable) */ +int LZ4IO_setSparseFile(int yes) +{ + sparseFile = yes; + return sparseFile; +} + +static int isSparse(const void* p, size_t size) +{ + const char* p8 = p; + for(; size; --size) { + if(*p8 != 0) { + return 0; + } + ++p8; + } + return 1; +} +#endif /* LZ4IO_ENABLE_SPARSE_FILE */ + /* blockSizeID : valid values : 4-5-6-7 */ int LZ4IO_setBlockSizeID(int bsid) { @@ -539,6 +572,9 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) LZ4F_decompressionContext_t ctx; LZ4F_errorCode_t errorCode; LZ4F_frameInfo_t frameInfo; +#if defined(LZ4IO_ENABLE_SPARSE_FILE) + size_t sparsePending = 0; +#endif /* init */ errorCode = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION); @@ -580,9 +616,30 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) filesize += decodedBytes; /* Write Block */ +#if defined(LZ4IO_ENABLE_SPARSE_FILE) + if(sparseFile) { + if(isSparse(outBuff, decodedBytes)) { + sparsePending += decodedBytes; + continue; + } + if(sparsePending > 0) { + fseek(foutput, sparsePending, SEEK_CUR); + sparsePending = 0; + } + } +#endif sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput); if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block\n"); } +#if defined(LZ4IO_ENABLE_SPARSE_FILE) + if(sparseFile) { + if(sparsePending > 0) { + fseek(foutput, sparsePending-1, SEEK_CUR); + fputc(0, foutput); + sparsePending = 0; + } + } +#endif /* Free */ free(inBuff); @@ -645,6 +702,14 @@ int LZ4IO_decompressFilename(char* input_filename, char* output_filename) start = clock(); get_fileHandle(input_filename, output_filename, &finput, &foutput); +#if defined(LZ4IO_ENABLE_SPARSE_FILE) + if (sparseFile!=0 && foutput!=0) + { + DISPLAY("Experimental : Using sparse file\n"); + SET_SPARSE_FILE_MODE(foutput); + } +#endif /* LZ4IO_ENABLE_SPARSE_FILE */ + /* Loop over multiple streams */ do { diff --git a/programs/lz4io.h b/programs/lz4io.h index 7869a43..a0c6119 100644 --- a/programs/lz4io.h +++ b/programs/lz4io.h @@ -75,3 +75,8 @@ int LZ4IO_setStreamChecksumMode(int xxhash); /* Default setting : 0 (no notification) */ int LZ4IO_setNotificationLevel(int level); + +#if defined(LZ4IO_ENABLE_SPARSE_FILE) +/* Default setting : 0 (sparseFile = 0; disable sparse file) */ +int LZ4IO_setSparseFile(int yes); +#endif -- cgit v0.12 From 4a5d92b1d9207b5d1309e1b21ecc1164cc14a149 Mon Sep 17 00:00:00 2001 From: Takayuki MATSUOKA Date: Thu, 12 Feb 2015 16:13:23 +0900 Subject: Adjust coding style --- programs/lz4io.c | 65 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index 4e583f6..9f80531 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -79,7 +79,9 @@ # if defined(LZ4IO_ENABLE_SPARSE_FILE) # include # define SET_SPARSE_FILE_MODE(file) do { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } while(0) -# define fseek _fseeki64 +# if defined(_MSC_VER) && (_MSC_VER >= 1400) +# define fseek _fseeki64 +# endif # endif /* LZ4IO_ENABLE_SPARSE_FILE */ #else # include /* isatty */ @@ -197,13 +199,15 @@ int LZ4IO_setSparseFile(int yes) static int isSparse(const void* p, size_t size) { const char* p8 = p; - for(; size; --size) { - if(*p8 != 0) { - return 0; - } - ++p8; - } - return 1; + for(; size; --size) + { + if(*p8 != 0) + { + return 0; + } + ++p8; + } + return 1; } #endif /* LZ4IO_ENABLE_SPARSE_FILE */ @@ -574,7 +578,7 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) LZ4F_frameInfo_t frameInfo; #if defined(LZ4IO_ENABLE_SPARSE_FILE) size_t sparsePending = 0; -#endif +#endif /* LZ4IO_ENABLE_SPARSE_FILE */ /* init */ errorCode = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION); @@ -617,29 +621,34 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) /* Write Block */ #if defined(LZ4IO_ENABLE_SPARSE_FILE) - if(sparseFile) { - if(isSparse(outBuff, decodedBytes)) { - sparsePending += decodedBytes; - continue; - } - if(sparsePending > 0) { - fseek(foutput, sparsePending, SEEK_CUR); - sparsePending = 0; - } - } -#endif + if(sparseFile) + { + if(isSparse(outBuff, decodedBytes)) + { + sparsePending += decodedBytes; + continue; + } + if(sparsePending > 0) + { + fseek(foutput, sparsePending, SEEK_CUR); + sparsePending = 0; + } + } +#endif /* LZ4IO_ENABLE_SPARSE_FILE */ sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput); if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block\n"); } #if defined(LZ4IO_ENABLE_SPARSE_FILE) - if(sparseFile) { - if(sparsePending > 0) { - fseek(foutput, sparsePending-1, SEEK_CUR); - fputc(0, foutput); - sparsePending = 0; - } - } -#endif + if(sparseFile) + { + if(sparsePending > 0) + { + fseek(foutput, sparsePending-1, SEEK_CUR); + fputc(0, foutput); + sparsePending = 0; + } + } +#endif /* LZ4IO_ENABLE_SPARSE_FILE */ /* Free */ free(inBuff); -- cgit v0.12 From 01a24afdcf867a901076eaa87434b3dcb39d526f Mon Sep 17 00:00:00 2001 From: Takayuki MATSUOKA Date: Thu, 12 Feb 2015 19:58:49 +0900 Subject: Improve isSparse() --- programs/lz4io.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/programs/lz4io.c b/programs/lz4io.c index 9f80531..b1067ac 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -62,6 +62,27 @@ #include "lz4frame.h" +/************************************** + Basic Types +**************************************/ +#if defined(LZ4IO_ENABLE_SPARSE_FILE) +#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif +#endif /* LZ4IO_ENABLE_SPARSE_FILE */ + + /**************************** * OS-specific Includes *****************************/ @@ -198,6 +219,7 @@ int LZ4IO_setSparseFile(int yes) static int isSparse(const void* p, size_t size) { +#if 0 const char* p8 = p; for(; size; --size) { @@ -208,6 +230,30 @@ static int isSparse(const void* p, size_t size) ++p8; } return 1; +#else + const U64* p64 = (const U64*) p; + const char* p8 = (const char*) p; + const size_t n = size / sizeof(*p64); + size_t i; + + for (i = 0; i < n; ++i) + { + if (p64[i] != 0) + { + return 0; + } + } + + for(i = n * sizeof(*p64); i < size; ++i) + { + if (p8[i] != 0) + { + return 0; + } + } + + return 1; +#endif } #endif /* LZ4IO_ENABLE_SPARSE_FILE */ -- cgit v0.12 From b372f45093aa8d86f7a99b7dd5e2deaeffb10033 Mon Sep 17 00:00:00 2001 From: Takayuki MATSUOKA Date: Thu, 12 Feb 2015 20:18:16 +0900 Subject: Add Neil's method to isSparse() --- programs/lz4io.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index b1067ac..c69275d 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -220,6 +220,7 @@ int LZ4IO_setSparseFile(int yes) static int isSparse(const void* p, size_t size) { #if 0 + /* naive */ const char* p8 = p; for(; size; --size) { @@ -230,7 +231,8 @@ static int isSparse(const void* p, size_t size) ++p8; } return 1; -#else +#elif 0 + /* xz method */ const U64* p64 = (const U64*) p; const char* p8 = (const char*) p; const size_t n = size / sizeof(*p64); @@ -253,6 +255,10 @@ static int isSparse(const void* p, size_t size) } return 1; +#else + /* Neil's */ + const char* buf = (const char*) p; + return buf[0] == 0 && !memcmp(buf, buf + 1, size - 1); #endif } #endif /* LZ4IO_ENABLE_SPARSE_FILE */ -- cgit v0.12 From eed7952101fbb740018cd00d7d2958d192da8344 Mon Sep 17 00:00:00 2001 From: Takayuki MATSUOKA Date: Thu, 12 Feb 2015 21:00:50 +0900 Subject: Add GNU coreutil's is_nul() method to isSparse() See original source http://git.savannah.gnu.org/cgit/coreutils.git/tree/src/system.h --- programs/lz4io.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index c69275d..7315d69 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -255,10 +255,24 @@ static int isSparse(const void* p, size_t size) } return 1; -#else +#elif 0 /* Neil's */ const char* buf = (const char*) p; return buf[0] == 0 && !memcmp(buf, buf + 1, size - 1); +#else + /* GNU Core Utilities : coreutils/src/system.h / is_nul() */ + const U64* wp = (const U64*) p; + const char* cbuf = (const char*) p; + const char* cp; + + // Find first nonzero *word*, or the word with the sentinel. + while(*wp++ == 0) ; + + // Find the first nonzero *byte*, or the sentinel. + cp = (const char*) (wp - 1); + while(*cp++ == 0) ; + + return cbuf + size < cp; #endif } #endif /* LZ4IO_ENABLE_SPARSE_FILE */ @@ -652,7 +666,12 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) outBuffSize = LZ4IO_setBlockSizeID(frameInfo.blockSizeID); inBuffSize = outBuffSize + 4; inBuff = (char*)malloc(inBuffSize); +#if defined(LZ4IO_ENABLE_SPARSE_FILE) + outBuff = (char*)malloc(outBuffSize+1); + outBuff[outBuffSize] = 1; /* sentinel */ +#else /* LZ4IO_ENABLE_SPARSE_FILE */ outBuff = (char*)malloc(outBuffSize); +#endif /* LZ4IO_ENABLE_SPARSE_FILE */ if (!inBuff || !outBuff) EXM_THROW(65, "Allocation error : not enough memory"); /* Main Loop */ -- cgit v0.12 From de5c930c901539082c57193b390fa3666a732883 Mon Sep 17 00:00:00 2001 From: Takayuki MATSUOKA Date: Thu, 12 Feb 2015 21:26:32 +0900 Subject: Fix sentinel size miscalculation Since is_nul() style wild-comparing stride is U64, we must add sizeof(U64) to the sentinel. --- programs/lz4io.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index 7315d69..03d93af 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -667,7 +667,7 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) inBuffSize = outBuffSize + 4; inBuff = (char*)malloc(inBuffSize); #if defined(LZ4IO_ENABLE_SPARSE_FILE) - outBuff = (char*)malloc(outBuffSize+1); + outBuff = (char*)malloc(outBuffSize+sizeof(U64)); outBuff[outBuffSize] = 1; /* sentinel */ #else /* LZ4IO_ENABLE_SPARSE_FILE */ outBuff = (char*)malloc(outBuffSize); -- cgit v0.12 From dcdd628d5ec8a78401dbe277151e7654e300eafc Mon Sep 17 00:00:00 2001 From: Takayuki MATSUOKA Date: Thu, 12 Feb 2015 23:50:14 +0900 Subject: Fix sentinel bit pattern Set all bits of U64. --- programs/lz4io.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index 03d93af..109db98 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -668,7 +668,7 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) inBuff = (char*)malloc(inBuffSize); #if defined(LZ4IO_ENABLE_SPARSE_FILE) outBuff = (char*)malloc(outBuffSize+sizeof(U64)); - outBuff[outBuffSize] = 1; /* sentinel */ + *(U64*) &outBuff[outBuffSize] = (U64) -1; /* sentinel */ #else /* LZ4IO_ENABLE_SPARSE_FILE */ outBuff = (char*)malloc(outBuffSize); #endif /* LZ4IO_ENABLE_SPARSE_FILE */ -- cgit v0.12 From 6b0c39b839b8343da195252a8c46e6d93138f3b8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 10 Mar 2015 00:12:51 +0100 Subject: Updated datagen (can create sparse files) --- Makefile | 2 +- programs/Makefile | 5 +- programs/datagen.c | 327 ++++++++++++++++++++------------------------------ programs/datagen.h | 40 ++++++ programs/datagencli.c | 190 +++++++++++++++++++++++++++++ 5 files changed, 366 insertions(+), 198 deletions(-) create mode 100644 programs/datagen.h create mode 100644 programs/datagencli.c diff --git a/Makefile b/Makefile index 5662cb4..5c7c69e 100644 --- a/Makefile +++ b/Makefile @@ -50,7 +50,7 @@ TEXT = $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4.h $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4hc.h \ Makefile lz4_block_format.txt LZ4_Frame_Format.html NEWS README.md \ cmake_unofficial/CMakeLists.txt \ $(PRGDIR)/fullbench.c $(PRGDIR)/lz4cli.c \ - $(PRGDIR)/datagen.c $(PRGDIR)/fuzzer.c \ + $(PRGDIR)/datagen.c $(PRGDIR)/datagen.h $(PRGDIR)/datagencli.c $(PRGDIR)/fuzzer.c \ $(PRGDIR)/lz4io.c $(PRGDIR)/lz4io.h \ $(PRGDIR)/bench.c $(PRGDIR)/bench.h \ $(PRGDIR)/lz4.1 $(PRGDIR)/lz4c.1 $(PRGDIR)/lz4cat.1 \ diff --git a/programs/Makefile b/programs/Makefile index 1070f40..2c883d0 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -92,7 +92,7 @@ frametest: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxha frametest32: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c frametest.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) -datagen : datagen.c +datagen : datagen.c datagencli.c $(CC) $(FLAGS) $^ -o $@$(EXT) @@ -142,7 +142,7 @@ test-travis: $(TRAVIS_TARGET) test-lz4: lz4 datagen ./datagen -g16KB | ./lz4 -9 | ./lz4 -vdq > $(VOID) ./datagen | ./lz4 | ./lz4 -vdq > $(VOID) - ./datagen -g6M -p100 | ./lz4 -9BD | ./lz4 -vdq > $(VOID) + ./datagen -g6M -P100 | ./lz4 -9BD | ./lz4 -vdq > $(VOID) ./datagen -g17M | ./lz4 -9v | ./lz4 -vdq > $(VOID) ./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -vdq > $(VOID) ./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -vdq > $(VOID) @@ -197,6 +197,7 @@ test-frametest32: frametest32 ./frametest32 test-mem: lz4 datagen fuzzer frametest + valgrind --leak-check=yes ./datagen -g50M > /dev/null ./datagen -g16KB > tmp valgrind --leak-check=yes ./lz4 -9 -BD -f tmp /dev/null ./datagen -g16MB > tmp diff --git a/programs/datagen.c b/programs/datagen.c index 0f07477..2a10b81 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -19,26 +19,20 @@ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4 - - LZ4 source mirror : https://github.com/Cyan4973/lz4 - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c + - ZSTD source repository : https://github.com/Cyan4973/zstd + - Public forum : https://groups.google.com/forum/#!forum/lz4c */ /************************************** - Remove Visual warning messages +* Includes **************************************/ -#define _CRT_SECURE_NO_WARNINGS // fgets +#include /* malloc */ +#include /* FILE, fwrite */ +#include /* memcpy */ /************************************** - Includes -**************************************/ -#include // fgets, sscanf -#include // strcmp - - -/************************************** - Basic Types +* Basic Types **************************************/ #if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ # include @@ -57,230 +51,173 @@ /************************************** - Constants +* OS-specific Includes **************************************/ -#ifndef LZ4_VERSION -# define LZ4_VERSION "r125" +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +# include /* _O_BINARY */ +# include /* _setmode, _isatty */ +# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) +#else +# define SET_BINARY_MODE(file) #endif -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -#define CDG_SIZE_DEFAULT (64 KB) -#define CDG_SEED_DEFAULT 0 -#define CDG_COMPRESSIBILITY_DEFAULT 50 -#define PRIME1 2654435761U -#define PRIME2 2246822519U - /************************************** - Macros +* Constants **************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } - +#define KB *(1 <<10) -/************************************** - Local Parameters -**************************************/ -static unsigned no_prompt = 0; -static char* programName; -static unsigned displayLevel = 2; +#define PRIME1 2654435761U +#define PRIME2 2246822519U /********************************************************* - functions +* Local Functions *********************************************************/ - -#define CDG_rotl32(x,r) ((x << r) | (x >> (32 - r))) -static unsigned int CDG_rand(U32* src) +#define RDG_rotl32(x,r) ((x << r) | (x >> (32 - r))) +static unsigned int RDG_rand(U32* src) { U32 rand32 = *src; rand32 *= PRIME1; - rand32 += PRIME2; - rand32 = CDG_rotl32(rand32, 13); + rand32 ^= PRIME2; + rand32 = RDG_rotl32(rand32, 13); *src = rand32; return rand32; } -#define CDG_RAND15BITS ((CDG_rand(seed) >> 3) & 32767) -#define CDG_RANDLENGTH ( ((CDG_rand(seed) >> 7) & 3) ? (CDG_rand(seed) % 14) : (CDG_rand(seed) & 511) + 15) -#define CDG_RANDCHAR (((CDG_rand(seed) >> 9) & 63) + '0') -static void CDG_generate(U64 size, U32* seed, double proba) +#define LTSIZE 8192 +#define LTMASK (LTSIZE-1) +static void* RDG_createLiteralDistrib(double ld) { - BYTE fullbuff[32 KB + 128 KB + 1]; - BYTE* buff = fullbuff + 32 KB; - U64 total=0; - U32 P32 = (U32)(32768 * proba); - U32 pos=1; - U32 genBlockSize = 128 KB; + BYTE* lt = malloc(LTSIZE); + U32 i = 0; + BYTE character = '0'; + BYTE firstChar = '('; + BYTE lastChar = '}'; - // Build initial prefix - fullbuff[0] = CDG_RANDCHAR; - while (pos<32 KB) + if (ld==0.0) { - // Select : Literal (char) or Match (within 32K) - if (CDG_RAND15BITS < P32) + character = 0; + firstChar = 0; + lastChar =255; + } + while (i LTSIZE) weight = LTSIZE-i; + end = i + weight; + while (i < end) lt[i++] = character; + character++; + if (character > lastChar) character = firstChar; + } + return lt; +} + +static char RDG_genChar(U32* seed, const void* ltctx) +{ + const BYTE* lt = ltctx; + U32 id = RDG_rand(seed) & LTMASK; + return lt[id]; +} + +#define RDG_DICTSIZE (32 KB) +#define RDG_RAND15BITS ((RDG_rand(seed) >> 3) & 32767) +#define RDG_RANDLENGTH ( ((RDG_rand(seed) >> 7) & 7) ? (RDG_rand(seed) & 15) : (RDG_rand(seed) & 511) + 15) +void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, void* litTable, unsigned* seedPtr) +{ + BYTE* buffPtr = (BYTE*)buffer; + const U32 matchProba32 = (U32)(32768 * matchProba); + size_t pos = prefixSize; + void* ldctx = litTable; + U32* seed = seedPtr; + + /* special case */ + while (matchProba >= 1.0) + { + size_t size0 = RDG_rand(seed) & 3; + size0 = 1U << (16 + size0 * 2); + size0 += RDG_rand(seed) & (size0-1); /* because size0 is power of 2*/ + if (buffSize < pos + size0) + { + memset(buffPtr+pos, 0, buffSize-pos); + return; + } + memset(buffPtr+pos, 0, size0); + pos += size0; + buffPtr[pos-1] = RDG_genChar(seed, ldctx); + } + + /* init */ + if (pos==0) buffPtr[0] = RDG_genChar(seed, ldctx), pos=1; + + /* Generate compressible data */ + while (pos < buffSize) + { + /* Select : Literal (char) or Match (within 32K) */ + if (RDG_RAND15BITS < matchProba32) { - // Copy (within 64K) + /* Copy (within 32K) */ + int match; U32 d; - int ref; - int length = CDG_RANDLENGTH + 4; - U32 offset = CDG_RAND15BITS + 1; + int length = RDG_RANDLENGTH + 4; + U32 offset = RDG_RAND15BITS + 1; if (offset > pos) offset = pos; - ref = pos - offset; + match = pos - offset; d = pos + length; - while (pos < d) fullbuff[pos++] = fullbuff[ref++]; + if (d > buffSize) d = buffSize; + while (pos < d) buffPtr[pos++] = buffPtr[match++]; } else { - // Literal (noise) - U32 d = pos + CDG_RANDLENGTH; - while (pos < d) fullbuff[pos++] = CDG_RANDCHAR; - } - } - - // Generate compressible data - pos = 0; - while (total < size) - { - if (size-total < 128 KB) genBlockSize = (U32)(size-total); - total += genBlockSize; - buff[genBlockSize] = 0; - pos = 0; - while (pos genBlockSize ) length = genBlockSize - pos; - ref = pos - offset; - d = pos + length; - while (pos < d) buff[pos++] = buff[ref++]; - } - else - { - // Literal (noise) - U32 d; - int length = CDG_RANDLENGTH; - if (pos + length > genBlockSize) length = genBlockSize - pos; - d = pos + length; - while (pos < d) buff[pos++] = CDG_RANDCHAR; - } + /* Literal (noise) */ + size_t d; + size_t length = RDG_RANDLENGTH; + d = pos + length; + if (d > buffSize) d = buffSize; + while (pos < d) buffPtr[pos++] = RDG_genChar(seed, ldctx); } - // output datagen - pos=0; - for (;pos+512<=genBlockSize;pos+=512) - printf("%512.512s", buff+pos); - for (;pos='0') && (*argument<='9')) - { - size *= 10; - size += *argument - '0'; - argument++; - } - if (*argument=='K') { size <<= 10; argument++; } - if (*argument=='M') { size <<= 20; argument++; } - if (*argument=='G') { size <<= 30; argument++; } - if (*argument=='B') { argument++; } - break; - case 's': - argument++; - seed=0; - while ((*argument>='0') && (*argument<='9')) - { - seed *= 10; - seed += *argument - '0'; - argument++; - } - break; - case 'p': - argument++; - proba=0; - while ((*argument>='0') && (*argument<='9')) - { - proba *= 10; - proba += *argument - '0'; - argument++; - } - if (proba<0) proba=0; - if (proba>100) proba=100; - break; - case 'v': - displayLevel = 4; - argument++; - break; - default: ; - } - } + /* Generate dict */ + RDG_genBlock(buff, RDG_DICTSIZE, 0, matchProba, ldctx, &seed); - } + /* Generate compressible data */ + while (total < size) + { + RDG_genBlock(buff, RDG_DICTSIZE+RDG_BLOCKSIZE, RDG_DICTSIZE, matchProba, ldctx, &seed); + if (size-total < RDG_BLOCKSIZE) genBlockSize = (size_t)(size-total); + total += genBlockSize; + fwrite(buff, 1, genBlockSize, stdout); + /* update dict */ + memcpy(buff, buff + RDG_BLOCKSIZE, RDG_DICTSIZE); } - // Get Seed - DISPLAYLEVEL(4, "Data Generator %s \n", LZ4_VERSION); - DISPLAYLEVEL(3, "Seed = %u \n", seed); - if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", proba); - - CDG_generate(size, &seed, ((double)proba) / 100); - - return 0; + free(ldctx); } diff --git a/programs/datagen.h b/programs/datagen.h new file mode 100644 index 0000000..631d146 --- /dev/null +++ b/programs/datagen.h @@ -0,0 +1,40 @@ +/* + datagen.h - compressible data generator header + Copyright (C) Yann Collet 2012-2015 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - ZSTD source repository : https://github.com/Cyan4973/zstd + - Public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + + +#include /* size_t */ + +void RDG_genOut(unsigned long long size, double matchProba, double litProba, unsigned seed); +void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed); +/* RDG_genOut + Generate 'size' bytes of compressible data into stdout. + Compressibility can be controlled using 'matchProba'. + 'LitProba' is optional, and affect variability of bytes. If litProba==0.0, default value is used. + Generated data can be selected using 'seed'. + If (matchProba, litProba and seed) are equal, the function always generate the same content. + + RDG_genBuffer + Same as RDG_genOut, but generate data into provided buffer +*/ diff --git a/programs/datagencli.c b/programs/datagencli.c new file mode 100644 index 0000000..801e198 --- /dev/null +++ b/programs/datagencli.c @@ -0,0 +1,190 @@ +/* + datagencli.c + compressible data command line generator + Copyright (C) Yann Collet 2012-2015 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - ZSTD source repository : https://github.com/Cyan4973/zstd + - Public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +/************************************** +* Includes +**************************************/ +#include /* fprintf, stderr */ +#include "datagen.h" /* RDG_generate */ + + +/************************************** +* Basic Types +**************************************/ +#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif + + +/************************************** +* Constants +**************************************/ +#ifndef ZSTD_VERSION +# define ZSTD_VERSION "r1" +#endif + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define SIZE_DEFAULT (64 KB) +#define SEED_DEFAULT 0 +#define COMPRESSIBILITY_DEFAULT 50 + + +/************************************** +* Macros +**************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static unsigned displayLevel = 2; + + +/********************************************************* +* Command line +*********************************************************/ +static int usage(char* programName) +{ + DISPLAY( "Compressible data generator\n"); + DISPLAY( "Usage :\n"); + DISPLAY( " %s [size] [args]\n", programName); + DISPLAY( "\n"); + DISPLAY( "Arguments :\n"); + DISPLAY( " -g# : generate # data (default:%i)\n", SIZE_DEFAULT); + DISPLAY( " -s# : Select seed (default:%i)\n", SEED_DEFAULT); + DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", COMPRESSIBILITY_DEFAULT); + DISPLAY( " -h : display help and exit\n"); + return 0; +} + + +int main(int argc, char** argv) +{ + int argNb; + double proba = (double)COMPRESSIBILITY_DEFAULT / 100; + double litProba = 0.0; + U64 size = SIZE_DEFAULT; + U32 seed = SEED_DEFAULT; + char* programName; + + /* Check command line */ + programName = argv[0]; + for(argNb=1; argNb='0') && (*argument<='9')) + { + size *= 10; + size += *argument - '0'; + argument++; + } + if (*argument=='K') { size <<= 10; argument++; } + if (*argument=='M') { size <<= 20; argument++; } + if (*argument=='G') { size <<= 30; argument++; } + if (*argument=='B') { argument++; } + break; + case 's': + argument++; + seed=0; + while ((*argument>='0') && (*argument<='9')) + { + seed *= 10; + seed += *argument - '0'; + argument++; + } + break; + case 'P': + argument++; + proba=0.0; + while ((*argument>='0') && (*argument<='9')) + { + proba *= 10; + proba += *argument - '0'; + argument++; + } + if (proba>100.) proba=100.; + proba /= 100.; + break; + case 'L': /* hidden argument : Literal distribution probability */ + argument++; + litProba=0.; + while ((*argument>='0') && (*argument<='9')) + { + litProba *= 10; + litProba += *argument - '0'; + argument++; + } + if (litProba>100.) litProba=100.; + litProba /= 100.; + break; + case 'v': + displayLevel = 4; + argument++; + break; + default: + return usage(programName); + } + } + + } + } + + DISPLAYLEVEL(4, "Data Generator %s \n", ZSTD_VERSION); + DISPLAYLEVEL(3, "Seed = %u \n", seed); + if (proba!=COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", (U32)(proba*100)); + + RDG_genOut(size, proba, litProba, seed); + DISPLAYLEVEL(1, "\n"); + + return 0; +} -- cgit v0.12 From ceec6fa8492a5ff0ed163c96516716a3c2b09461 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 10 Mar 2015 16:57:42 +0100 Subject: g++ compatibility --- .travis.yml | 2 + Makefile | 4 ++ NEWS | 2 + lib/lz4.c | 10 +-- lib/lz4frame.c | 92 ++++++++++++-------------- lib/lz4frame.h | 2 +- programs/bench.c | 11 ++-- programs/bench.h | 17 ++--- programs/datagen.c | 4 +- programs/frametest.c | 11 ++-- programs/fullbench.c | 2 +- programs/fuzzer.c | 2 +- programs/lz4cli.c | 29 ++------- programs/lz4io.c | 178 ++++----------------------------------------------- programs/lz4io.h | 6 +- 15 files changed, 92 insertions(+), 280 deletions(-) mode change 100755 => 100644 programs/fullbench.c diff --git a/.travis.yml b/.travis.yml index 9eff075..650fba1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ compiler: gcc script: make test-travis before_install: - sudo apt-get update -qq + - sudo apt-get install -qq g++-multilib - sudo apt-get install -qq gcc-multilib - sudo apt-get install -qq valgrind @@ -10,6 +11,7 @@ env: - LZ4_TRAVIS_CI_ENV=travis-install - LZ4_TRAVIS_CI_ENV=streaming-examples - LZ4_TRAVIS_CI_ENV=cmake + - LZ4_TRAVIS_CI_ENV=gpptest - LZ4_TRAVIS_CI_ENV=dist - LZ4_TRAVIS_CI_ENV=test-lz4 - LZ4_TRAVIS_CI_ENV=test-lz4c diff --git a/Makefile b/Makefile index 5c7c69e..65bbfde 100644 --- a/Makefile +++ b/Makefile @@ -127,6 +127,10 @@ test-travis: $(TRAVIS_TARGET) cmake: @cd cmake_unofficial; cmake CMakeLists.txt; $(MAKE) +gpptest: + export CC=g++; export CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align"; $(MAKE) -e all + + streaming-examples: cd examples; $(MAKE) -e test diff --git a/NEWS b/NEWS index dbea778..1de6466 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,8 @@ r128: New : command -m, to compress multiple files in a single command Fixed : Restored lz4hc compression ratio (slightly lower since r124) +New : g++ compatibility test +New : datagen can generate sparse files Fixed : Fuzzer + frametest compatibility with NetBSD (issue #48) Added : Visual project directory diff --git a/lib/lz4.c b/lib/lz4.c index a8b4531..d1a2849 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -190,7 +190,7 @@ static U16 LZ4_readLE16(const void* memPtr) return *(U16*)memPtr; else { - const BYTE* p = memPtr; + const BYTE* p = (const BYTE*)memPtr; return (U16)((U16)p[0] + (p[1]<<8)); } } @@ -204,7 +204,7 @@ static void LZ4_writeLE16(void* memPtr, U16 value) } else { - BYTE* p = memPtr; + BYTE* p = (BYTE*)memPtr; p[0] = (BYTE) value; p[1] = (BYTE)(value>>8); } @@ -285,9 +285,9 @@ static void LZ4_copy8(void* dstPtr, const void* srcPtr) /* customized version of memcpy, which may overwrite up to 7 bytes beyond dstEnd */ static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) { - BYTE* d = dstPtr; - const BYTE* s = srcPtr; - BYTE* e = dstEnd; + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* e = (BYTE*)dstEnd; do { LZ4_copy8(d,s); d+=8; s+=8; } while (d /* malloc, calloc, free */ #define ALLOCATOR(s) calloc(1,s) @@ -62,7 +62,7 @@ Memory routines /************************************** -Includes +* Includes **************************************/ #include "lz4frame_static.h" #include "lz4.h" @@ -71,7 +71,7 @@ Includes /************************************** -Basic Types +* Basic Types **************************************/ #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ # include @@ -90,7 +90,7 @@ typedef unsigned long long U64; /************************************** -Constants +* Constants **************************************/ #define KB *(1<<10) #define MB *(1<<20) @@ -110,7 +110,7 @@ Constants static const U32 minHClevel = 3; /************************************** -Structures and local types +* Structures and local types **************************************/ typedef struct { @@ -150,12 +150,7 @@ typedef struct /************************************** -Macros -**************************************/ - - -/************************************** -Error management +* Error management **************************************/ #define LZ4F_GENERATE_STRING(STRING) #STRING, static const char* LZ4F_errorStrings[] = { LZ4F_LIST_ERRORS(LZ4F_GENERATE_STRING) }; @@ -175,7 +170,7 @@ const char* LZ4F_getErrorName(LZ4F_errorCode_t code) /************************************** -Private functions +* Private functions **************************************/ static size_t LZ4F_getBlockSize(unsigned blockSizeID) { @@ -215,14 +210,15 @@ static BYTE LZ4F_headerChecksum (const BYTE* header, size_t length) /************************************** -Simple compression functions +* Simple compression functions **************************************/ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { - LZ4F_preferences_t prefs = { 0 }; + LZ4F_preferences_t prefs; size_t headerSize; size_t streamSize; + memset(&prefs, 0, sizeof(prefs)); if (preferencesPtr!=NULL) prefs = *preferencesPtr; { blockSizeID_t proposedBSID = max64KB; @@ -234,7 +230,7 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere prefs.frameInfo.blockSizeID = proposedBSID; break; } - proposedBSID++; + proposedBSID = (blockSizeID_t)( ((int)proposedBSID) + 1); maxBlockSize <<= 2; } } @@ -258,14 +254,17 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere */ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { - LZ4F_cctx_internal_t cctxI = { 0 }; /* works because no allocation */ - LZ4F_preferences_t prefs = { 0 }; - LZ4F_compressOptions_t options = { 0 }; + LZ4F_cctx_internal_t cctxI; + LZ4F_preferences_t prefs; + LZ4F_compressOptions_t options; LZ4F_errorCode_t errorCode; BYTE* const dstStart = (BYTE*) dstBuffer; BYTE* dstPtr = dstStart; BYTE* const dstEnd = dstStart + dstMaxSize; + memset(&cctxI, 0, sizeof(cctxI)); /* works because no allocation */ + memset(&prefs, 0, sizeof(prefs)); + memset(&options, 0, sizeof(options)); cctxI.version = LZ4F_VERSION; cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent allocation; works because autoflush==1 & stableSrc==1 */ @@ -281,7 +280,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf prefs.frameInfo.blockSizeID = proposedBSID; break; } - proposedBSID++; + proposedBSID = (blockSizeID_t)((int)proposedBSID + 1); maxBlockSize <<= 2; } } @@ -361,7 +360,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp */ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_preferences_t* preferencesPtr) { - LZ4F_preferences_t prefNull = { 0 }; + LZ4F_preferences_t prefNull; LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; @@ -370,6 +369,7 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds if (dstMaxSize < LZ4F_MAXHEADERFRAME_SIZE) return (size_t)-ERROR_dstMaxSize_tooSmall; if (cctxPtr->cStage != 0) return (size_t)-ERROR_GENERIC; + memset(&prefNull, 0, sizeof(prefNull)); if (preferencesPtr == NULL) preferencesPtr = &prefNull; cctxPtr->prefs = *preferencesPtr; @@ -436,7 +436,7 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds * */ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { - const LZ4F_preferences_t prefsNull = { 0 }; + const LZ4F_preferences_t prefsNull = {}; /* init to zero */ const LZ4F_preferences_t* prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID; size_t blockSize = LZ4F_getBlockSize(bid); @@ -444,9 +444,8 @@ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesP size_t lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize; size_t blockInfo = 4; /* default, without block CRC option */ size_t frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); - size_t result = (blockInfo * nbBlocks) + (blockSize * (nbBlocks-1)) + lastBlockSize + frameEnd; - return result; + return (blockInfo * nbBlocks) + (blockSize * (nbBlocks-1)) + lastBlockSize + frameEnd;; } @@ -712,7 +711,7 @@ LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_compressionContext_t* LZ4F { LZ4F_dctx_internal_t* dctxPtr; - dctxPtr = ALLOCATOR(sizeof(LZ4F_dctx_internal_t)); + dctxPtr = (LZ4F_dctx_internal_t*)ALLOCATOR(sizeof(LZ4F_dctx_internal_t)); if (dctxPtr==NULL) return (LZ4F_errorCode_t)-ERROR_GENERIC; dctxPtr->version = versionNumber; @@ -732,11 +731,12 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_compressionContext_t LZ4F_de /* Decompression */ -static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPtr, size_t srcSize) +static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const void* srcVoidPtr, size_t srcSize) { BYTE FLG, BD, HC; unsigned version, blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, dictFlag, blockSizeID; size_t bufferNeeded; + const BYTE* srcPtr = (const BYTE*)srcVoidPtr; /* need to decode header to get frameInfo */ if (srcSize < 7) return (size_t)-ERROR_GENERIC; /* minimal header size */ @@ -747,7 +747,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPt /* Flags */ FLG = srcPtr[0]; - version = (FLG>>6)&_2BITS; + version = (FLG>>6) & _2BITS; blockMode = (FLG>>5) & _1BIT; blockChecksumFlag = (FLG>>4) & _1BIT; contentSizeFlag = (FLG>>3) & _1BIT; @@ -761,19 +761,19 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPt if (HC != srcPtr[2]) return (size_t)-ERROR_GENERIC; /* Bad header checksum error */ /* validate */ - if (version != 1) return (size_t)-ERROR_GENERIC; /* Version Number, only supported value */ - if (blockChecksumFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */ + if (version != 1) return (size_t)-ERROR_GENERIC; /* Version Number, only supported value */ + if (blockChecksumFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */ if (contentSizeFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */ - if (((FLG>>1)&_1BIT) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bit */ - if (dictFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */ + if (((FLG>>1)&_1BIT) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bit */ + if (dictFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */ if (((BD>>7)&_1BIT) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bit */ - if (blockSizeID < 4) return (size_t)-ERROR_GENERIC; /* Only supported values for the time being */ - if (((BD>>0)&_4BITS) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bits */ + if (blockSizeID < 4) return (size_t)-ERROR_GENERIC; /* 4-7 only supported values for the time being */ + if (((BD>>0)&_4BITS) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bits */ /* save */ - dctxPtr->frameInfo.blockMode = blockMode; - dctxPtr->frameInfo.contentChecksumFlag = contentChecksumFlag; - dctxPtr->frameInfo.blockSizeID = blockSizeID; + dctxPtr->frameInfo.blockMode = (blockMode_t)blockMode; + dctxPtr->frameInfo.contentChecksumFlag = (contentChecksum_t)contentChecksumFlag; + dctxPtr->frameInfo.blockSizeID = (blockSizeID_t)blockSizeID; dctxPtr->maxBlockSize = LZ4F_getBlockSize(blockSizeID); /* init */ @@ -786,9 +786,9 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPt FREEMEM(dctxPtr->tmpIn); FREEMEM(dctxPtr->tmpOutBuffer); dctxPtr->maxBufferSize = bufferNeeded; - dctxPtr->tmpIn = ALLOCATOR(dctxPtr->maxBlockSize); + dctxPtr->tmpIn = (BYTE*)ALLOCATOR(dctxPtr->maxBlockSize); if (dctxPtr->tmpIn == NULL) return (size_t)-ERROR_GENERIC; - dctxPtr->tmpOutBuffer= ALLOCATOR(dctxPtr->maxBufferSize); + dctxPtr->tmpOutBuffer= (BYTE*)ALLOCATOR(dctxPtr->maxBufferSize); if (dctxPtr->tmpOutBuffer== NULL) return (size_t)-ERROR_GENERIC; } dctxPtr->tmpInSize = 0; @@ -880,15 +880,6 @@ static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, s if (withinTmp) /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */ { -#if 0 - size_t savedDictSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; - memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart- savedDictSize, savedDictSize); - dctxPtr->dict = dctxPtr->tmpOutBuffer; - dctxPtr->dictSize = savedDictSize + dctxPtr->tmpOutStart + dstSize; - return; - -#else - size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; size_t copySize = 64 KB - dctxPtr->tmpOutSize; BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; @@ -900,7 +891,6 @@ static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, s dctxPtr->dict = dctxPtr->tmpOutBuffer; dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart + dstSize; return; -#endif } if (dctxPtr->dict == dctxPtr->tmpOutBuffer) /* copy dst into tmp to complete dict */ diff --git a/lib/lz4frame.h b/lib/lz4frame.h index d73e3e2..f52ed2f 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -215,7 +215,7 @@ size_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t ctx, * The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). * You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr) * The function result is an hint of how many srcSize bytes LZ4F_decompress() expects for next call, - * or an error code which can be tested using LZ4F_isError(). + * or an error code which can be tested using LZ4F_isError(). */ size_t LZ4F_decompress(LZ4F_decompressionContext_t ctx, diff --git a/programs/bench.c b/programs/bench.c index 77120f2..0ed7fcb 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -1,6 +1,7 @@ /* - bench.c - Demo program to benchmark open-source compression algorithm + bench.c - Demo program to benchmark open-source compression algorithms Copyright (C) Yann Collet 2012-2015 + GPL v2 License This program is free software; you can redistribute it and/or modify @@ -141,15 +142,15 @@ static int chunkSize = DEFAULT_CHUNKSIZE; static int nbIterations = NBLOOPS; static int BMK_pause = 0; -void BMK_SetBlocksize(int bsize) { chunkSize = bsize; } +void BMK_setBlocksize(int bsize) { chunkSize = bsize; } -void BMK_SetNbIterations(int nbLoops) +void BMK_setNbIterations(int nbLoops) { nbIterations = nbLoops; DISPLAY("- %i iterations -\n", nbIterations); } -void BMK_SetPause(void) { BMK_pause = 1; } +void BMK_setPause(void) { BMK_pause = 1; } /********************************************************* @@ -234,7 +235,7 @@ static U64 BMK_GetFileSize(const char* infilename) * Public function **********************************************************/ -int BMK_benchFile(const char** fileNamesTable, int nbFiles, int cLevel) +int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel) { int fileIdx=0; char* orig_buff; diff --git a/programs/bench.h b/programs/bench.h index 2a20cdb..3231727 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -22,21 +22,12 @@ */ #pragma once -#if defined (__cplusplus) -extern "C" { -#endif - /* Main function */ -int BMK_benchFile(const char** fileNamesTable, int nbFiles, int cLevel); +int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel); /* Set Parameters */ -void BMK_SetBlocksize(int bsize); -void BMK_SetNbIterations(int nbLoops); -void BMK_SetPause(void); - - +void BMK_setBlocksize(int bsize); +void BMK_setNbIterations(int nbLoops); +void BMK_setPause(void); -#if defined (__cplusplus) -} -#endif diff --git a/programs/datagen.c b/programs/datagen.c index 2a10b81..743691e 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -90,7 +90,7 @@ static unsigned int RDG_rand(U32* src) #define LTMASK (LTSIZE-1) static void* RDG_createLiteralDistrib(double ld) { - BYTE* lt = malloc(LTSIZE); + BYTE* lt = (BYTE*)malloc(LTSIZE); U32 i = 0; BYTE character = '0'; BYTE firstChar = '('; @@ -117,7 +117,7 @@ static void* RDG_createLiteralDistrib(double ld) static char RDG_genChar(U32* seed, const void* ltctx) { - const BYTE* lt = ltctx; + const BYTE* lt = (const BYTE*)ltctx; U32 id = RDG_rand(seed) & LTMASK; return lt[id]; } diff --git a/programs/frametest.c b/programs/frametest.c index a5a23c2..df37e5f 100644 --- a/programs/frametest.c +++ b/programs/frametest.c @@ -229,11 +229,12 @@ int basicTests(U32 seed, double compressibility) void* decodedBuffer; U32 randState = seed; size_t cSize, testSize; - LZ4F_preferences_t prefs = { 0 }; + LZ4F_preferences_t prefs; LZ4F_decompressionContext_t dCtx; U64 crcOrig; // Create compressible test buffer + memset(&prefs, 0, sizeof(prefs)); CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); compressedBuffer = malloc(LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL)); decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); @@ -441,7 +442,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi unsigned BMId = FUZ_rand(&randState) & 1; unsigned CCflag = FUZ_rand(&randState) & 1; unsigned autoflush = (FUZ_rand(&randState) & 7) == 2; - LZ4F_preferences_t prefs = { 0 }; + LZ4F_preferences_t prefs; LZ4F_compressOptions_t cOptions = { 0 }; LZ4F_decompressOptions_t dOptions = { 0 }; unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(srcDataLength-1) - 1)) + 1; @@ -452,6 +453,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi LZ4F_preferences_t* prefsPtr = &prefs; (void)FUZ_rand(&coreRand); // update rand seed + memset(&prefs, 0, sizeof(prefs)); prefs.frameInfo.blockMode = (blockMode_t)BMId; prefs.frameInfo.blockSizeID = (blockSizeID_t)BSId; prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)CCflag; @@ -523,17 +525,14 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi if (oSize > (size_t)(oend-op)) oSize = oend-op; dOptions.stableDst = FUZ_rand(&randState) & 1; if (nonContiguousDst==2) dOptions.stableDst = 0; - //if (ip == compressedBuffer+62073) DISPLAY("oSize : %i : pos %i \n", (int)oSize, (int)(op-(BYTE*)decodedBuffer)); result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions); - //if (op+oSize >= (BYTE*)decodedBuffer+94727) DISPLAY("iSize : %i : pos %i \n", (int)iSize, (int)(ip-(BYTE*)compressedBuffer)); - //if ((int)result<0) DISPLAY("iSize : %i : pos %i \n", (int)iSize, (int)(ip-(BYTE*)compressedBuffer)); if (result == (size_t)-ERROR_checksum_invalid) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst); CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s)", (int)result, LZ4F_getErrorName((LZ4F_errorCode_t)result)); XXH64_update(&xxh64, op, (U32)oSize); op += oSize; ip += iSize; op += nonContiguousDst; - if (nonContiguousDst==2) op = decodedBuffer; // overwritten destination + if (nonContiguousDst==2) op = (BYTE*)decodedBuffer; /* overwritten destination */ } CHECK(result != 0, "Frame decompression failed (error %i)", (int)result); crcDecoded = XXH64_digest(&xxh64); diff --git a/programs/fullbench.c b/programs/fullbench.c old mode 100755 new mode 100644 index 756357a..6ae28d7 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -687,7 +687,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) milliTime = BMK_GetMilliStart(); while(BMK_GetMilliSpan(milliTime) < TIMELOOP) { - if (initFunction!=NULL) ctx = initFunction(chunkP[0].origBuffer); + if (initFunction!=NULL) ctx = (LZ4_stream_t*)initFunction(chunkP[0].origBuffer); for (chunkNb=0; chunkNb='1') && (argument[1] <='9')) { int iters = argument[1] - '0'; - BMK_SetNbIterations(iters); + BMK_setNbIterations(iters); argument++; } break; /* Pause at the end (hidden option) */ - case 'p': main_pause=1; BMK_SetPause(); break; - -#if defined(LZ4IO_ENABLE_SPARSE_FILE) - /* Experimental : Enable sparse file */ - case 'x': LZ4IO_setSparseFile(1); break; -#endif /* LZ4IO_ENABLE_SPARSE_FILE */ + case 'p': main_pause=1; BMK_setPause(); break; /* Specific commands for customized versions */ EXTENDED_ARGUMENTS; @@ -471,7 +454,7 @@ int main(int argc, char** argv) if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage(); /* Check if benchmark is selected */ - if (bench) return BMK_benchFile(inFileNames, ifnIdx, cLevel); + if (bench) return BMK_benchFiles(inFileNames, ifnIdx, cLevel); /* No output filename ==> try to select one automatically (when possible) */ while (!output_filename) diff --git a/programs/lz4io.c b/programs/lz4io.c index 9d5bee5..fd98247 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -39,20 +39,15 @@ #endif #define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#ifdef __GNUC__ -# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ -# pragma GCC diagnostic ignored "-Wmissing-field-initializers" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ -#endif #define _LARGE_FILES /* Large file support on 32-bits AIX */ #define _FILE_OFFSET_BITS 64 /* Large file support on 32-bits unix */ -#define _POSIX_SOURCE 1 /* for fileno() within on unix */ /**************************** * Includes *****************************/ -#include /* fprintf, fopen, fread, _fileno, stdin, stdout */ +#include /* fprintf, fopen, fread, stdin, stdout */ #include /* malloc, free */ #include /* strcmp, strlen */ #include /* clock */ @@ -62,55 +57,18 @@ #include "lz4frame.h" -/************************************** - Basic Types -**************************************/ -#if defined(LZ4IO_ENABLE_SPARSE_FILE) -#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif -#endif /* LZ4IO_ENABLE_SPARSE_FILE */ - - /**************************** * OS-specific Includes *****************************/ #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) # include /* _O_BINARY */ -# include /* _setmode, _isatty */ -# ifdef __MINGW32__ - int _fileno(FILE *stream); /* MINGW somehow forgets to include this windows declaration into */ -# endif +# include /* _setmode, _fileno */ +# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) # if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ # define fseek _fseeki64 # endif -# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) -# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) -# if defined(LZ4IO_ENABLE_SPARSE_FILE) -# include -# define SET_SPARSE_FILE_MODE(file) do { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } while(0) -# if defined(_MSC_VER) && (_MSC_VER >= 1400) -# define fseek _fseeki64 -# endif -# endif /* LZ4IO_ENABLE_SPARSE_FILE */ #else -# include /* isatty */ # define SET_BINARY_MODE(file) -# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) -# if defined(LZ4IO_ENABLE_SPARSE_FILE) -# define SET_SPARSE_FILE_MODE(file) -# endif /* LZ4IO_ENABLE_SPARSE_FILE */ #endif @@ -164,9 +122,6 @@ static int globalBlockSizeId = LZ4S_BLOCKSIZEID_DEFAULT; static int blockChecksum = 0; static int streamChecksum = 1; static int blockIndependence = 1; -#if defined(LZ4IO_ENABLE_SPARSE_FILE) -static int sparseFile = 0; -#endif /* LZ4IO_ENABLE_SPARSE_FILE */ static const int minBlockSizeID = 4; static const int maxBlockSizeID = 7; @@ -193,7 +148,6 @@ static const int maxBlockSizeID = 7; #define EXTENDED_ARGUMENTS #define EXTENDED_HELP #define EXTENDED_FORMAT -#define DEFAULT_COMPRESSOR compress_file #define DEFAULT_DECOMPRESSOR decodeLZ4S @@ -208,75 +162,6 @@ int LZ4IO_setOverwrite(int yes) return overwrite; } -#if defined(LZ4IO_ENABLE_SPARSE_FILE) -/* Default setting : sparseFile = 0; (Disable) - return : sparse file mode (0:Disable / 1:Enable) */ -int LZ4IO_setSparseFile(int yes) -{ - sparseFile = yes; - return sparseFile; -} - -static int isSparse(const void* p, size_t size) -{ -#if 0 - /* naive */ - const char* p8 = p; - for(; size; --size) - { - if(*p8 != 0) - { - return 0; - } - ++p8; - } - return 1; -#elif 0 - /* xz method */ - const U64* p64 = (const U64*) p; - const char* p8 = (const char*) p; - const size_t n = size / sizeof(*p64); - size_t i; - - for (i = 0; i < n; ++i) - { - if (p64[i] != 0) - { - return 0; - } - } - - for(i = n * sizeof(*p64); i < size; ++i) - { - if (p8[i] != 0) - { - return 0; - } - } - - return 1; -#elif 0 - /* Neil's */ - const char* buf = (const char*) p; - return buf[0] == 0 && !memcmp(buf, buf + 1, size - 1); -#else - /* GNU Core Utilities : coreutils/src/system.h / is_nul() */ - const U64* wp = (const U64*) p; - const char* cbuf = (const char*) p; - const char* cp; - - // Find first nonzero *word*, or the word with the sentinel. - while(*wp++ == 0) ; - - // Find the first nonzero *byte*, or the sentinel. - cp = (const char*) (wp - 1); - while(*cp++ == 0) ; - - return cbuf + size < cp; -#endif -} -#endif /* LZ4IO_ENABLE_SPARSE_FILE */ - /* blockSizeID : valid values : 4-5-6-7 */ int LZ4IO_setBlockSizeID(int bsid) { @@ -386,7 +271,7 @@ static int get_fileHandle(const char* input_filename, const char* output_filenam /* unoptimized version; solves endianess & alignment issues */ static void LZ4IO_writeLE32 (void* p, unsigned value32) { - unsigned char* dstPtr = p; + unsigned char* dstPtr = (unsigned char*)p; dstPtr[0] = (unsigned char)value32; dstPtr[1] = (unsigned char)(value32 >> 8); dstPtr[2] = (unsigned char)(value32 >> 16); @@ -483,11 +368,12 @@ int LZ4IO_compressFilename(const char* input_filename, const char* output_filena size_t sizeCheck, headerSize, readSize, outBuffSize; LZ4F_compressionContext_t ctx; LZ4F_errorCode_t errorCode; - LZ4F_preferences_t prefs = {0}; + LZ4F_preferences_t prefs; /* Init */ start = clock(); + memset(&prefs, 0, sizeof(prefs)); if ((displayLevel==2) && (compressionLevel>=3)) displayLevel=3; errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode)); @@ -497,9 +383,9 @@ int LZ4IO_compressFilename(const char* input_filename, const char* output_filena /* Set compression parameters */ prefs.autoFlush = 1; prefs.compressionLevel = compressionLevel; - prefs.frameInfo.blockMode = blockIndependence; - prefs.frameInfo.blockSizeID = globalBlockSizeId; - prefs.frameInfo.contentChecksumFlag = streamChecksum; + prefs.frameInfo.blockMode = (blockMode_t)blockIndependence; + prefs.frameInfo.blockSizeID = (blockSizeID_t)globalBlockSizeId; + prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)streamChecksum; /* Allocate Memory */ in_buff = (char*)malloc(blockSize); @@ -578,7 +464,7 @@ int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, for (i=0; i 0) - { - fseek(foutput, sparsePending, SEEK_CUR); - sparsePending = 0; - } - } -#endif /* LZ4IO_ENABLE_SPARSE_FILE */ sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput); if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block\n"); } -#if defined(LZ4IO_ENABLE_SPARSE_FILE) - if(sparseFile) - { - if(sparsePending > 0) - { - fseek(foutput, sparsePending-1, SEEK_CUR); - fputc(0, foutput); - sparsePending = 0; - } - } -#endif /* LZ4IO_ENABLE_SPARSE_FILE */ /* Free */ free(inBuff); @@ -803,14 +655,6 @@ int LZ4IO_decompressFilename(const char* input_filename, const char* output_file start = clock(); get_fileHandle(input_filename, output_filename, &finput, &foutput); -#if defined(LZ4IO_ENABLE_SPARSE_FILE) - if (sparseFile!=0 && foutput!=0) - { - DISPLAY("Experimental : Using sparse file\n"); - SET_SPARSE_FILE_MODE(foutput); - } -#endif /* LZ4IO_ENABLE_SPARSE_FILE */ - /* Loop over multiple streams */ do { diff --git a/programs/lz4io.h b/programs/lz4io.h index 1c9b837..f99e8bb 100644 --- a/programs/lz4io.h +++ b/programs/lz4io.h @@ -29,6 +29,7 @@ - The license of this source file is GPLv2. */ +#pragma once /* ************************************************** */ /* Special input/output values */ @@ -77,8 +78,3 @@ int LZ4IO_setStreamChecksumMode(int xxhash); /* Default setting : 0 (no notification) */ int LZ4IO_setNotificationLevel(int level); - -#if defined(LZ4IO_ENABLE_SPARSE_FILE) -/* Default setting : 0 (sparseFile = 0; disable sparse file) */ -int LZ4IO_setSparseFile(int yes); -#endif -- cgit v0.12 From e3f33d25793193e005eadf4c04c2fbcf2eae497e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 11 Mar 2015 18:38:44 +0100 Subject: Fixed minor warnings --- lib/lz4.c | 2 +- lib/lz4frame.c | 23 +++++++++++++---------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index d1a2849..f21270a 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -1151,7 +1151,7 @@ typedef struct */ LZ4_streamDecode_t* LZ4_createStreamDecode(void) { - LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(sizeof(U64), LZ4_STREAMDECODESIZE_U64); + LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(LZ4_STREAMDECODESIZE_U64, sizeof(U64)); return lz4s; } diff --git a/lib/lz4frame.c b/lib/lz4frame.c index d81690f..2218fd2 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -436,16 +436,19 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds * */ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { - const LZ4F_preferences_t prefsNull = {}; /* init to zero */ - const LZ4F_preferences_t* prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; - blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID; - size_t blockSize = LZ4F_getBlockSize(bid); - unsigned nbBlocks = (unsigned)(srcSize / blockSize) + 1; - size_t lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize; - size_t blockInfo = 4; /* default, without block CRC option */ - size_t frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); - - return (blockInfo * nbBlocks) + (blockSize * (nbBlocks-1)) + lastBlockSize + frameEnd;; + LZ4F_preferences_t prefsNull; + memset(&prefsNull, 0, sizeof(prefsNull)); + { + const LZ4F_preferences_t* prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; + blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID; + size_t blockSize = LZ4F_getBlockSize(bid); + unsigned nbBlocks = (unsigned)(srcSize / blockSize) + 1; + size_t lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize; + size_t blockInfo = 4; /* default, without block CRC option */ + size_t frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); + + return (blockInfo * nbBlocks) + (blockSize * (nbBlocks-1)) + lastBlockSize + frameEnd;; + } } -- cgit v0.12 From 12ab41571ef7fd11b8b2013aa943beae373cef8a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 11 Mar 2015 19:42:37 +0100 Subject: Preliminary support for sparse files --- programs/Makefile | 9 ++++-- programs/lz4cli.c | 4 +++ programs/lz4io.c | 91 ++++++++++++++++++++++++++++++++++--------------------- programs/lz4io.h | 8 +++-- 4 files changed, 74 insertions(+), 38 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index 2c883d0..b9bb5b3 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -143,10 +143,11 @@ test-lz4: lz4 datagen ./datagen -g16KB | ./lz4 -9 | ./lz4 -vdq > $(VOID) ./datagen | ./lz4 | ./lz4 -vdq > $(VOID) ./datagen -g6M -P100 | ./lz4 -9BD | ./lz4 -vdq > $(VOID) - ./datagen -g17M | ./lz4 -9v | ./lz4 -vdq > $(VOID) + ./datagen -g17M | ./lz4 -9v | ./lz4 -dq > $(VOID) ./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -vdq > $(VOID) ./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -vdq > $(VOID) # test frame concatenation with null-length frame + @echo *** test frame concatenation *** @echo -n > empty.test @echo hi > nonempty.test cat nonempty.test empty.test nonempty.test > orig.test @@ -158,12 +159,16 @@ test-lz4: lz4 datagen @rm *.test @echo frame concatenation test completed # test frame concatenation with null-length frame - @echo test multiple input files + @echo *** test multiple input files *** @./datagen -s1 > file1 @./datagen -s2 > file2 @./datagen -s3 > file3 ./lz4 -f -m file1 file2 file3 @rm file1 file2 file3 file1.lz4 file2.lz4 file3.lz4 + @echo *** test sparse file support *** + ./datagen -g50M -P100 | ./lz4 -B4 | ./lz4 -dvX > tmp + ls -ls tmp + @rm tmp test-lz4c: lz4c datagen diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 10b980f..7ecfa93 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -176,6 +176,7 @@ static int usage_advanced(void) DISPLAY( " -BD : Block dependency (improve compression ratio)\n"); /* DISPLAY( " -BX : enable block checksum (default:disabled)\n"); *//* Option currently inactive */ DISPLAY( " -Sx : disable stream checksum (default:enabled)\n"); + DISPLAY( " -X : enable sparse file (default:disabled)(experimental)\n"); DISPLAY( "Benchmark arguments :\n"); DISPLAY( " -b : benchmark file(s)\n"); DISPLAY( " -i# : iteration loops [1-9](default : 3), benchmark mode only\n"); @@ -390,6 +391,9 @@ int main(int argc, char** argv) /* Modify Stream properties */ case 'S': if (argument[1]=='x') { LZ4IO_setStreamChecksumMode(0); argument++; break; } else { badusage(); } + /* Enable Sparse File support (experimental) */ + case 'X': LZ4IO_setSparseFile(1); break; + /* Benchmark */ case 'b': bench=1; multiple_inputs=1; if (inFileNames == NULL) diff --git a/programs/lz4io.c b/programs/lz4io.c index fd98247..6e977f1 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -104,11 +104,13 @@ * Macros ***************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } -#define DISPLAYUPDATE(l, ...) if (displayLevel>=l) { \ - if ((LZ4IO_GetMilliSpan(g_time) > refreshRate) || (displayLevel>=4)) \ +#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static int g_displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */ + +#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ + if ((LZ4IO_GetMilliSpan(g_time) > refreshRate) || (g_displayLevel>=4)) \ { g_time = clock(); DISPLAY(__VA_ARGS__); \ - if (displayLevel>=4) fflush(stdout); } } + if (g_displayLevel>=4) fflush(stdout); } } static const unsigned refreshRate = 150; static clock_t g_time = 0; @@ -116,12 +118,12 @@ static clock_t g_time = 0; /************************************** * Local Parameters ***************************************/ -static int displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */ -static int overwrite = 1; -static int globalBlockSizeId = LZ4S_BLOCKSIZEID_DEFAULT; -static int blockChecksum = 0; -static int streamChecksum = 1; -static int blockIndependence = 1; +static int g_overwrite = 1; +static int g_blockSizeId = LZ4S_BLOCKSIZEID_DEFAULT; +static int g_blockChecksum = 0; +static int g_streamChecksum = 1; +static int g_blockIndependence = 1; +static int g_sparseFileSupport = 0; static const int minBlockSizeID = 4; static const int maxBlockSizeID = 7; @@ -158,8 +160,8 @@ static const int maxBlockSizeID = 7; /* Default setting : overwrite = 1; return : overwrite mode (0/1) */ int LZ4IO_setOverwrite(int yes) { - overwrite = (yes!=0); - return overwrite; + g_overwrite = (yes!=0); + return g_overwrite; } /* blockSizeID : valid values : 4-5-6-7 */ @@ -167,35 +169,42 @@ int LZ4IO_setBlockSizeID(int bsid) { static const int blockSizeTable[] = { 64 KB, 256 KB, 1 MB, 4 MB }; if ((bsid < minBlockSizeID) || (bsid > maxBlockSizeID)) return -1; - globalBlockSizeId = bsid; - return blockSizeTable[globalBlockSizeId-minBlockSizeID]; + g_blockSizeId = bsid; + return blockSizeTable[g_blockSizeId-minBlockSizeID]; } int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode) { - blockIndependence = (blockMode == LZ4IO_blockIndependent); - return blockIndependence; + g_blockIndependence = (blockMode == LZ4IO_blockIndependent); + return g_blockIndependence; } /* Default setting : no checksum */ int LZ4IO_setBlockChecksumMode(int xxhash) { - blockChecksum = (xxhash != 0); - return blockChecksum; + g_blockChecksum = (xxhash != 0); + return g_blockChecksum; } /* Default setting : checksum enabled */ int LZ4IO_setStreamChecksumMode(int xxhash) { - streamChecksum = (xxhash != 0); - return streamChecksum; + g_streamChecksum = (xxhash != 0); + return g_streamChecksum; } /* Default setting : 0 (no notification) */ int LZ4IO_setNotificationLevel(int level) { - displayLevel = level; - return displayLevel; + g_displayLevel = level; + return g_displayLevel; +} + +/* Default setting : 0 (disabled) */ +int LZ4IO_setSparseFile(int enable) +{ + g_sparseFileSupport = enable; + return g_sparseFileSupport; } static unsigned LZ4IO_GetMilliSpan(clock_t nPrevious) @@ -242,12 +251,12 @@ static int get_fileHandle(const char* input_filename, const char* output_filenam if (*pfoutput!=0) { fclose(*pfoutput); - if (!overwrite) + if (!g_overwrite) { char ch; DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename); DISPLAYLEVEL(2, "Overwrite ? (Y/N) : "); - if (displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); /* No interaction possible */ + if (g_displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); /* No interaction possible */ ch = (char)getchar(); if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); } @@ -299,7 +308,7 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output if (compressionlevel < 3) compressionFunction = LZ4_compress; else compressionFunction = LZ4_compressHC; get_fileHandle(input_filename, output_filename, &finput, &foutput); - if ((displayLevel==2) && (compressionlevel==1)) displayLevel=3; + if ((g_displayLevel==2) && (compressionlevel==1)) g_displayLevel=3; /* Allocate Memory */ in_buff = (char*)malloc(LEGACY_BLOCKSIZE); @@ -374,18 +383,18 @@ int LZ4IO_compressFilename(const char* input_filename, const char* output_filena /* Init */ start = clock(); memset(&prefs, 0, sizeof(prefs)); - if ((displayLevel==2) && (compressionLevel>=3)) displayLevel=3; + if ((g_displayLevel==2) && (compressionLevel>=3)) g_displayLevel=3; errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode)); get_fileHandle(input_filename, output_filename, &finput, &foutput); - blockSize = LZ4S_GetBlockSize_FromBlockId (globalBlockSizeId); + blockSize = LZ4S_GetBlockSize_FromBlockId (g_blockSizeId); /* Set compression parameters */ prefs.autoFlush = 1; prefs.compressionLevel = compressionLevel; - prefs.frameInfo.blockMode = (blockMode_t)blockIndependence; - prefs.frameInfo.blockSizeID = (blockSizeID_t)globalBlockSizeId; - prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)streamChecksum; + prefs.frameInfo.blockMode = (blockMode_t)g_blockIndependence; + prefs.frameInfo.blockSizeID = (blockSizeID_t)g_blockSizeId; + prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)g_streamChecksum; /* Allocate Memory */ in_buff = (char*)malloc(blockSize); @@ -541,8 +550,8 @@ static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput) static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) { unsigned long long filesize = 0; - char* inBuff; - char* outBuff; + void* inBuff; + void* outBuff; # define HEADERMAX 20 char headerBuff[HEADERMAX]; size_t sizeCheck, nextToRead, outBuffSize, inBuffSize; @@ -569,8 +578,8 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) /* Allocate Memory */ outBuffSize = LZ4IO_setBlockSizeID(frameInfo.blockSizeID); inBuffSize = outBuffSize + 4; - inBuff = (char*)malloc(inBuffSize); - outBuff = (char*)malloc(outBuffSize); + inBuff = malloc(inBuffSize); + outBuff = malloc(outBuffSize); if (!inBuff || !outBuff) EXM_THROW(65, "Allocation error : not enough memory"); /* Main Loop */ @@ -590,6 +599,20 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) filesize += decodedBytes; /* Write Block */ + if (g_sparseFileSupport) + { + size_t* sPtr = (size_t*)outBuff; + size_t toCheckLength = decodedBytes / sizeof(size_t); + size_t checked; + size_t skippedLength; + int seekResult; + for (checked=0; (checked < toCheckLength) && (sPtr[checked] == 0); checked++) ; + skippedLength = checked * sizeof(size_t); + if (skippedLength == decodedBytes) skippedLength--; /* ensure 1 byte at least is written */ + seekResult = fseek(foutput, skippedLength, SEEK_CUR); + if (seekResult != 0) EXM_THROW(68, "Skip error (sparse file)\n"); + decodedBytes -= skippedLength; + } sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput); if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block\n"); } diff --git a/programs/lz4io.h b/programs/lz4io.h index f99e8bb..12c9a6a 100644 --- a/programs/lz4io.h +++ b/programs/lz4io.h @@ -70,11 +70,15 @@ int LZ4IO_setBlockSizeID(int blockSizeID); typedef enum { LZ4IO_blockLinked=0, LZ4IO_blockIndependent} LZ4IO_blockMode_t; int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode); -/* Default setting : no checksum */ +/* Default setting : no block checksum */ int LZ4IO_setBlockChecksumMode(int xxhash); -/* Default setting : checksum enabled */ +/* Default setting : stream checksum enabled */ int LZ4IO_setStreamChecksumMode(int xxhash); /* Default setting : 0 (no notification) */ int LZ4IO_setNotificationLevel(int level); + +/* Default setting : 0 (disabled) */ +int LZ4IO_setSparseFile(int enable); + -- cgit v0.12 From e38c268b5a4b9289c29ad4d11c8f81c13b52464c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 11 Mar 2015 20:11:14 +0100 Subject: Fixed minor g++ warnings --- Makefile | 2 +- NEWS | 1 + lib/lz4frame.c | 13 ++++++------- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 65bbfde..b10e4d7 100644 --- a/Makefile +++ b/Makefile @@ -82,7 +82,7 @@ clean: @rm -f $(DISTRIBNAME) *.sha1 @cd $(PRGDIR); $(MAKE) clean @cd $(LZ4DIR); $(MAKE) clean - @cd examples; $(MAKE) clean + @cd examples; $(MAKE) clean @echo Cleaning completed diff --git a/NEWS b/NEWS index 1de6466..09247c0 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ r128: +New : lz4 cli sparse file support New : command -m, to compress multiple files in a single command Fixed : Restored lz4hc compression ratio (slightly lower since r124) New : g++ compatibility test diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 2218fd2..6feb5dc 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -46,9 +46,6 @@ Compiler Options #endif #define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#ifdef __GNUC__ -# pragma GCC diagnostic ignored "-Wmissing-field-initializers" /* GCC bug 53119 : doesn't accept {0} nor {} as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ -#endif /************************************** @@ -520,7 +517,7 @@ typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus; */ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr) { - LZ4F_compressOptions_t cOptionsNull = { 0 }; + LZ4F_compressOptions_t cOptionsNull; LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; size_t blockSize = cctxPtr->maxBlockSize; const BYTE* srcPtr = (const BYTE*)srcBuffer; @@ -533,6 +530,7 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d if (cctxPtr->cStage != 1) return (size_t)-ERROR_GENERIC; if (dstMaxSize < LZ4F_compressBound(srcSize, &(cctxPtr->prefs))) return (size_t)-ERROR_dstMaxSize_tooSmall; + memset(&cOptionsNull, 0, sizeof(cOptionsNull)); if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; /* select compression function */ @@ -629,7 +627,7 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d */ size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) { - LZ4F_compressOptions_t cOptionsNull = { 0 }; + LZ4F_compressOptions_t cOptionsNull; LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; @@ -639,8 +637,8 @@ size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */ if (cctxPtr->cStage != 1) return (size_t)-ERROR_GENERIC; if (dstMaxSize < (cctxPtr->tmpInSize + 16)) return (size_t)-ERROR_dstMaxSize_tooSmall; + memset(&cOptionsNull, 0, sizeof(cOptionsNull)); if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; - (void)compressOptionsPtr; /* not yet useful */ /* select compression function */ compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); @@ -945,7 +943,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, const LZ4F_decompressOptions_t* decompressOptionsPtr) { LZ4F_dctx_internal_t* dctxPtr = (LZ4F_dctx_internal_t*)decompressionContext; - static const LZ4F_decompressOptions_t optionsNull = { 0 }; + LZ4F_decompressOptions_t optionsNull; const BYTE* const srcStart = (const BYTE*)srcBuffer; const BYTE* const srcEnd = srcStart + *srcSizePtr; const BYTE* srcPtr = srcStart; @@ -957,6 +955,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, size_t nextSrcSizeHint = 1; + memset(&optionsNull, 0, sizeof(optionsNull)); if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull; *srcSizePtr = 0; *dstSizePtr = 0; -- cgit v0.12 From 45a357fd1704e9c6d2d8037277bda62e8c86308e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 13 Mar 2015 02:24:08 +0100 Subject: Improved sparse file support --- lib/lz4.c | 3 +-- lib/lz4.h | 5 +++-- lib/lz4frame.c | 4 ++-- lib/lz4frame.h | 3 +-- lib/lz4frame_static.h | 3 +-- lib/lz4hc.c | 7 ++++--- lib/lz4hc.h | 6 +++--- lib/xxhash.c | 3 +-- programs/Makefile | 12 ++++++------ programs/bench.c | 4 ++-- programs/bench.h | 4 ++-- programs/datagen.c | 8 ++++---- programs/lz4cli.c | 2 +- programs/lz4io.c | 42 +++++++++++++++++++++++++++++++++++------- programs/lz4io.h | 2 +- 15 files changed, 67 insertions(+), 41 deletions(-) mode change 100644 => 100755 programs/datagen.c diff --git a/lib/lz4.c b/lib/lz4.c index f21270a..a651843 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -27,8 +27,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4 - - LZ4 source mirror : https://github.com/Cyan4973/lz4 + - LZ4 source repository : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ diff --git a/lib/lz4.h b/lib/lz4.h index 7778caa..df1d839 100644 --- a/lib/lz4.h +++ b/lib/lz4.h @@ -1,7 +1,8 @@ /* LZ4 - Fast LZ compression algorithm Header File - Copyright (C) 2011-2014, Yann Collet. + Copyright (C) 2011-2015, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -28,7 +29,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 source repository : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ #pragma once diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 6feb5dc..7153871 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -1,6 +1,6 @@ /* LZ4 auto-framing library -Copyright (C) 2011-2014, Yann Collet. +Copyright (C) 2011-2015, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -28,7 +28,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : -- LZ4 source repository : http://code.google.com/p/lz4/ +- LZ4 source repository : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ diff --git a/lib/lz4frame.h b/lib/lz4frame.h index f52ed2f..61e461b 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -28,8 +28,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ - - LZ4 source mirror : https://github.com/Cyan4973/lz4 + - LZ4 source repository : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h index cde8186..4c34c6c 100644 --- a/lib/lz4frame_static.h +++ b/lib/lz4frame_static.h @@ -29,8 +29,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ - - LZ4 source mirror : https://github.com/Cyan4973/lz4 + - LZ4 source repository : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 5549969..357fa96 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -1,6 +1,7 @@ /* LZ4 HC - High Compression Mode of LZ4 -Copyright (C) 2011-2014, Yann Collet. +Copyright (C) 2011-2015, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -27,8 +28,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : -- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html -- LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 source repository : https://github.com/Cyan4973/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ diff --git a/lib/lz4hc.h b/lib/lz4hc.h index ce813ab..eb72051 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -1,7 +1,7 @@ /* LZ4 HC - High Compression Mode of LZ4 Header File - Copyright (C) 2011-2014, Yann Collet. + Copyright (C) 2011-2015, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -28,8 +28,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 source repository : https://github.com/Cyan4973/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ #pragma once diff --git a/lib/xxhash.c b/lib/xxhash.c index 093564c..aca1e0a 100644 --- a/lib/xxhash.c +++ b/lib/xxhash.c @@ -28,8 +28,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : -- xxHash source repository : http://code.google.com/p/xxhash/ -- xxHash source mirror : https://github.com/Cyan4973/xxHash +- xxHash source repository : https://github.com/Cyan4973/xxHash - public discussion board : https://groups.google.com/forum/#!forum/lz4c */ diff --git a/programs/Makefile b/programs/Makefile index b9bb5b3..f6fbd68 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -146,8 +146,7 @@ test-lz4: lz4 datagen ./datagen -g17M | ./lz4 -9v | ./lz4 -dq > $(VOID) ./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -vdq > $(VOID) ./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -vdq > $(VOID) -# test frame concatenation with null-length frame - @echo *** test frame concatenation *** + @echo ---- test frame concatenation ---- @echo -n > empty.test @echo hi > nonempty.test cat nonempty.test empty.test nonempty.test > orig.test @@ -158,15 +157,16 @@ test-lz4: lz4 datagen sdiff orig.test result.test @rm *.test @echo frame concatenation test completed -# test frame concatenation with null-length frame - @echo *** test multiple input files *** + @echo ---- test multiple input files ---- @./datagen -s1 > file1 @./datagen -s2 > file2 @./datagen -s3 > file3 ./lz4 -f -m file1 file2 file3 + ls -l file* @rm file1 file2 file3 file1.lz4 file2.lz4 file3.lz4 - @echo *** test sparse file support *** - ./datagen -g50M -P100 | ./lz4 -B4 | ./lz4 -dvX > tmp + @echo ---- test sparse file support ---- + ./datagen -g50M -P100 | ./lz4 -B4D | ./lz4 -dvX > tmp + ./datagen -g50M -P100 | diff -s - tmp ls -ls tmp @rm tmp diff --git a/programs/bench.c b/programs/bench.c index 0ed7fcb..b632314 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -19,8 +19,8 @@ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. You can contact the author at : - - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 source repository : https://github.com/Cyan4973/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ /************************************** diff --git a/programs/bench.h b/programs/bench.h index 3231727..c04fb17 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -17,8 +17,8 @@ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ - - LZ4 public forum : https://group.google.com/forum/#!forum/lz4c + - LZ4 source repository : https://github.com/Cyan4973/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ #pragma once diff --git a/programs/datagen.c b/programs/datagen.c old mode 100644 new mode 100755 index 743691e..bccb21e --- a/programs/datagen.c +++ b/programs/datagen.c @@ -137,7 +137,7 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match while (matchProba >= 1.0) { size_t size0 = RDG_rand(seed) & 3; - size0 = 1U << (16 + size0 * 2); + size0 = (size_t)1 << (16 + size0 * 2); size0 += RDG_rand(seed) & (size0-1); /* because size0 is power of 2*/ if (buffSize < pos + size0) { @@ -159,11 +159,11 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match if (RDG_RAND15BITS < matchProba32) { /* Copy (within 32K) */ - int match; - U32 d; + size_t match; + size_t d; int length = RDG_RANDLENGTH + 4; U32 offset = RDG_RAND15BITS + 1; - if (offset > pos) offset = pos; + if (offset > pos) offset = (U32)pos; match = pos - offset; d = pos + length; if (d > buffSize) d = buffSize; diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 7ecfa93..28b7fd6 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -19,7 +19,7 @@ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 source repository : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ /* diff --git a/programs/lz4io.c b/programs/lz4io.c index 6e977f1..ebd270e 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -1,6 +1,7 @@ /* LZ4io.c - LZ4 File/Stream Interface Copyright (C) Yann Collet 2011-2015 + GPL v2 License This program is free software; you can redistribute it and/or modify @@ -18,7 +19,7 @@ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 source repository : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ /* @@ -558,6 +559,7 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) LZ4F_decompressionContext_t ctx; LZ4F_errorCode_t errorCode; LZ4F_frameInfo_t frameInfo; + unsigned storedSkips = 0; /* init */ errorCode = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION); @@ -608,13 +610,39 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) int seekResult; for (checked=0; (checked < toCheckLength) && (sPtr[checked] == 0); checked++) ; skippedLength = checked * sizeof(size_t); - if (skippedLength == decodedBytes) skippedLength--; /* ensure 1 byte at least is written */ - seekResult = fseek(foutput, skippedLength, SEEK_CUR); - if (seekResult != 0) EXM_THROW(68, "Skip error (sparse file)\n"); - decodedBytes -= skippedLength; + storedSkips += (unsigned)skippedLength; + if (storedSkips > 2 GB) + { + seekResult = fseek(foutput, 2 GB, SEEK_CUR); + if (seekResult != 0) EXM_THROW(68, "2 GB skip error (sparse file)\n"); + storedSkips -= 2 GB; + } + if (skippedLength != decodedBytes) + { + seekResult = fseek(foutput, storedSkips, SEEK_CUR); + if (seekResult != 0) EXM_THROW(68, "Skip error (sparse file)\n"); + storedSkips = 0; + decodedBytes -= skippedLength; + sizeCheck = fwrite(((char*)outBuff) + skippedLength, 1, decodedBytes, foutput); + if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block\n"); + } } - sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput); - if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block\n"); + else + { + sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput); + if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block\n"); + } + } + + if ((g_sparseFileSupport) && (storedSkips>0)) + { + int seekResult; + storedSkips --; + seekResult = fseek(foutput, storedSkips, SEEK_CUR); + if (seekResult != 0) EXM_THROW(69, "Skip error (sparse file)\n"); + memset(outBuff, 0, 1); + sizeCheck = fwrite(outBuff, 1, 1, foutput); + if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write decoded block\n"); } /* Free */ diff --git a/programs/lz4io.h b/programs/lz4io.h index 12c9a6a..2441174 100644 --- a/programs/lz4io.h +++ b/programs/lz4io.h @@ -18,7 +18,7 @@ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 source repository : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ /* -- cgit v0.12 From d11ac4087254db4b9391039ce862e7305874fe97 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 13 Mar 2015 20:36:59 +0100 Subject: Improved sparse files support --- programs/Makefile | 14 ++++++--- programs/datagencli.c | 3 ++ programs/lz4io.c | 87 +++++++++++++++++++++++++++++---------------------- 3 files changed, 62 insertions(+), 42 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index f6fbd68..3d38244 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -165,10 +165,16 @@ test-lz4: lz4 datagen ls -l file* @rm file1 file2 file3 file1.lz4 file2.lz4 file3.lz4 @echo ---- test sparse file support ---- - ./datagen -g50M -P100 | ./lz4 -B4D | ./lz4 -dvX > tmp - ./datagen -g50M -P100 | diff -s - tmp - ls -ls tmp - @rm tmp + ./datagen -g50M -P100 | ./lz4 -B4D | ./lz4 -dvX > tmpB4 + ./datagen -g50M -P100 | ./lz4 -B5D | ./lz4 -dvX > tmpB5 + ./datagen -g50M -P100 | ./lz4 -B6D | ./lz4 -dvX > tmpB6 + ./datagen -g50M -P100 | ./lz4 -B7D | ./lz4 -dvX > tmpB7 + ls -ls tmp* + ./datagen -g50M -P100 | diff -s - tmpB4 + ./datagen -g50M -P100 | diff -s - tmpB5 + ./datagen -g50M -P100 | diff -s - tmpB6 + ./datagen -g50M -P100 | diff -s - tmpB7 + @rm tmp* test-lz4c: lz4c datagen diff --git a/programs/datagencli.c b/programs/datagencli.c index 801e198..601cb0a 100644 --- a/programs/datagencli.c +++ b/programs/datagencli.c @@ -88,6 +88,9 @@ static int usage(char* programName) DISPLAY( " -s# : Select seed (default:%i)\n", SEED_DEFAULT); DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", COMPRESSIBILITY_DEFAULT); DISPLAY( " -h : display help and exit\n"); + DISPLAY( "Special values :\n"); + DISPLAY( " -P0 : generate incompressible noise\n"); + DISPLAY( " -P100 : generate sparse files\n"); return 0; } diff --git a/programs/lz4io.c b/programs/lz4io.c index ebd270e..762bdb5 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -32,20 +32,18 @@ /************************************** * Compiler Options -***************************************/ +**************************************/ #ifdef _MSC_VER /* Visual Studio */ # define _CRT_SECURE_NO_WARNINGS # define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ #endif -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - #define _LARGE_FILES /* Large file support on 32-bits AIX */ #define _FILE_OFFSET_BITS 64 /* Large file support on 32-bits unix */ -/**************************** +/***************************** * Includes *****************************/ #include /* fprintf, fopen, fread, stdin, stdout */ @@ -58,7 +56,7 @@ #include "lz4frame.h" -/**************************** +/***************************** * OS-specific Includes *****************************/ #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) @@ -103,7 +101,7 @@ /************************************** * Macros -***************************************/ +**************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } static int g_displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */ @@ -118,7 +116,7 @@ static clock_t g_time = 0; /************************************** * Local Parameters -***************************************/ +**************************************/ static int g_overwrite = 1; static int g_blockSizeId = LZ4S_BLOCKSIZEID_DEFAULT; static int g_blockChecksum = 0; @@ -216,9 +214,9 @@ static unsigned LZ4IO_GetMilliSpan(clock_t nPrevious) } -/* ************************************************************************ */ -/* ********************** LZ4 File / Pipe compression ********************* */ -/* ************************************************************************ */ +/* ************************************************************************ ** +** ********************** LZ4 File / Pipe compression ********************* ** +** ************************************************************************ */ static int LZ4S_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } static int LZ4S_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4S_SKIPPABLEMASK) == LZ4S_SKIPPABLE0; } @@ -275,8 +273,8 @@ static int get_fileHandle(const char* input_filename, const char* output_filenam /*************************************** - * Legacy Compression - * *************************************/ +* Legacy Compression +***************************************/ /* unoptimized version; solves endianess & alignment issues */ static void LZ4IO_writeLE32 (void* p, unsigned value32) @@ -361,9 +359,9 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output } -/*********************************************** - * Compression using Frame format - * ********************************************/ +/********************************************* +* Compression using Frame format +*********************************************/ int LZ4IO_compressFilename(const char* input_filename, const char* output_filename, int compressionLevel) { @@ -603,28 +601,41 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) /* Write Block */ if (g_sparseFileSupport) { - size_t* sPtr = (size_t*)outBuff; - size_t toCheckLength = decodedBytes / sizeof(size_t); - size_t checked; - size_t skippedLength; - int seekResult; - for (checked=0; (checked < toCheckLength) && (sPtr[checked] == 0); checked++) ; - skippedLength = checked * sizeof(size_t); - storedSkips += (unsigned)skippedLength; - if (storedSkips > 2 GB) - { - seekResult = fseek(foutput, 2 GB, SEEK_CUR); - if (seekResult != 0) EXM_THROW(68, "2 GB skip error (sparse file)\n"); - storedSkips -= 2 GB; - } - if (skippedLength != decodedBytes) + char* const oBuffStart = (char*)outBuff; + char* oBuffPos = oBuffStart; + char* const oBuffEnd = oBuffStart + decodedBytes; + static const size_t zeroBlockSize = 32 KB; + while (oBuffPos < oBuffEnd) { - seekResult = fseek(foutput, storedSkips, SEEK_CUR); - if (seekResult != 0) EXM_THROW(68, "Skip error (sparse file)\n"); - storedSkips = 0; - decodedBytes -= skippedLength; - sizeCheck = fwrite(((char*)outBuff) + skippedLength, 1, decodedBytes, foutput); - if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block\n"); + size_t* sPtr = (size_t*)oBuffPos; + size_t seg0Size = zeroBlockSize; + size_t nbSizeT; + size_t checked; + size_t skippedLength; + int seekResult; + if (seg0Size > decodedBytes) seg0Size = decodedBytes; + decodedBytes -= seg0Size; + nbSizeT = seg0Size / sizeof(size_t); + for (checked=0; (checked < nbSizeT) && (sPtr[checked] == 0); checked++) ; + skippedLength = checked * sizeof(size_t); + storedSkips += (unsigned)skippedLength; + if (storedSkips > 1 GB) + { + seekResult = fseek(foutput, 1 GB, SEEK_CUR); + if (seekResult != 0) EXM_THROW(68, "1 GB skip error (sparse file)\n"); + storedSkips -= 1 GB; + } + if (skippedLength != seg0Size) + { + seekResult = fseek(foutput, storedSkips, SEEK_CUR); + if (seekResult != 0) EXM_THROW(68, "Skip error (sparse file)\n"); + storedSkips = 0; + seg0Size -= skippedLength; + oBuffPos += skippedLength; + sizeCheck = fwrite(oBuffPos, 1, seg0Size, foutput); + if (sizeCheck != seg0Size) EXM_THROW(68, "Write error : cannot write decoded block\n"); + } + oBuffPos += seg0Size; } } else @@ -639,10 +650,10 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) int seekResult; storedSkips --; seekResult = fseek(foutput, storedSkips, SEEK_CUR); - if (seekResult != 0) EXM_THROW(69, "Skip error (sparse file)\n"); + if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n"); memset(outBuff, 0, 1); sizeCheck = fwrite(outBuff, 1, 1, foutput); - if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write decoded block\n"); + if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n"); } /* Free */ -- cgit v0.12 From 248b761f55b92a34eacbfe2ac5d6f1b94e69d7bf Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 13 Mar 2015 22:15:08 +0100 Subject: windows sparse file support --- programs/lz4io.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) mode change 100644 => 100755 programs/lz4io.c diff --git a/programs/lz4io.c b/programs/lz4io.c old mode 100644 new mode 100755 index 762bdb5..94bf5ea --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -56,22 +56,25 @@ #include "lz4frame.h" -/***************************** +/****************************** * OS-specific Includes -*****************************/ +******************************/ #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) # include /* _O_BINARY */ -# include /* _setmode, _fileno */ +# include /* _setmode, _fileno, _get_osfhandle */ # define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) +# include /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ +# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } # if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ # define fseek _fseeki64 # endif #else # define SET_BINARY_MODE(file) +# define SET_SPARSE_FILE_MODE(file) #endif -/**************************** +/***************************** * Constants *****************************/ #define KB *(1 <<10) @@ -607,7 +610,7 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) static const size_t zeroBlockSize = 32 KB; while (oBuffPos < oBuffEnd) { - size_t* sPtr = (size_t*)oBuffPos; + const size_t* sPtr = (const size_t*)oBuffPos; size_t seg0Size = zeroBlockSize; size_t nbSizeT; size_t checked; @@ -717,6 +720,12 @@ int LZ4IO_decompressFilename(const char* input_filename, const char* output_file start = clock(); get_fileHandle(input_filename, output_filename, &finput, &foutput); + /* sparse file */ + if (g_sparseFileSupport && foutput) + { + SET_SPARSE_FILE_MODE(foutput); + } + /* Loop over multiple streams */ do { -- cgit v0.12