diff options
author | Yann Collet <yann.collet.73@gmail.com> | 2015-03-23 07:48:42 (GMT) |
---|---|---|
committer | Yann Collet <yann.collet.73@gmail.com> | 2015-03-23 07:48:42 (GMT) |
commit | 00c3208c7a352f4136300b257b418c55f07450c8 (patch) | |
tree | 51466bdcec0166a2e25d4dc276abac6f284d7f72 /programs/lz4io.c | |
parent | e25b51de7b51101e04ceea194dd557fcc23c03ca (diff) | |
parent | 7f436a1215f11b0fb872c34f088b8b5888d0630d (diff) | |
download | lz4-00c3208c7a352f4136300b257b418c55f07450c8.zip lz4-00c3208c7a352f4136300b257b418c55f07450c8.tar.gz lz4-00c3208c7a352f4136300b257b418c55f07450c8.tar.bz2 |
Merge pull request #61 from Cyan4973/dev
Dev
Diffstat (limited to 'programs/lz4io.c')
-rw-r--r-- | programs/lz4io.c | 419 |
1 files changed, 290 insertions, 129 deletions
diff --git a/programs/lz4io.c b/programs/lz4io.c index fa1f0f9..f5c5e98 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -1,6 +1,7 @@ /* LZ4io.c - LZ4 File/Stream Interface - Copyright (C) Yann Collet 2011-2014 + Copyright (C) Yann Collet 2011-2015 + GPL v2 License This program is free software; you can redistribute it and/or modify @@ -18,7 +19,7 @@ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 source repository : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ /* @@ -31,56 +32,51 @@ /************************************** * Compiler Options -***************************************/ +**************************************/ #ifdef _MSC_VER /* Visual Studio */ # define _CRT_SECURE_NO_WARNINGS # define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ #endif -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#ifdef __GNUC__ -# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ -# pragma GCC diagnostic ignored "-Wmissing-field-initializers" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ -#endif - #define _LARGE_FILES /* Large file support on 32-bits AIX */ #define _FILE_OFFSET_BITS 64 /* Large file support on 32-bits unix */ -#define _POSIX_SOURCE 1 /* for fileno() within <stdio.h> on unix */ -/**************************** +/***************************** * Includes *****************************/ -#include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */ -#include <stdlib.h> /* malloc, free */ -#include <string.h> /* strcmp, strlen */ -#include <time.h> /* clock */ +#include <stdio.h> /* fprintf, fopen, fread, stdin, stdout */ +#include <stdlib.h> /* malloc, free */ +#include <string.h> /* strcmp, strlen */ +#include <time.h> /* clock */ +#include <sys/types.h> /* stat64 */ +#include <sys/stat.h> /* stat64 */ #include "lz4io.h" #include "lz4.h" /* still required for legacy format */ #include "lz4hc.h" /* still required for legacy format */ #include "lz4frame.h" -/**************************** +/****************************** * OS-specific Includes -*****************************/ +******************************/ #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) # include <fcntl.h> /* _O_BINARY */ -# include <io.h> /* _setmode, _isatty */ -# ifdef __MINGW32__ - int _fileno(FILE *stream); /* MINGW somehow forgets to include this windows declaration into <stdio.h> */ -# endif +# include <io.h> /* _setmode, _fileno, _get_osfhandle */ # define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) -# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +# include <Windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ +# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } +# if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ +# define fseek _fseeki64 +# endif #else -# include <unistd.h> /* isatty */ # define SET_BINARY_MODE(file) -# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +# define SET_SPARSE_FILE_MODE(file) #endif -/**************************** +/***************************** * Constants *****************************/ #define KB *(1 <<10) @@ -93,43 +89,46 @@ #define _4BITS 0x0F #define _8BITS 0xFF -#define MAGICNUMBER_SIZE 4 -#define LZ4S_MAGICNUMBER 0x184D2204 -#define LZ4S_SKIPPABLE0 0x184D2A50 -#define LZ4S_SKIPPABLEMASK 0xFFFFFFF0 -#define LEGACY_MAGICNUMBER 0x184C2102 +#define MAGICNUMBER_SIZE 4 +#define LZ4IO_MAGICNUMBER 0x184D2204 +#define LZ4IO_SKIPPABLE0 0x184D2A50 +#define LZ4IO_SKIPPABLEMASK 0xFFFFFFF0 +#define LEGACY_MAGICNUMBER 0x184C2102 #define CACHELINE 64 #define LEGACY_BLOCKSIZE (8 MB) #define MIN_STREAM_BUFSIZE (192 KB) -#define LZ4S_BLOCKSIZEID_DEFAULT 7 -#define LZ4S_CHECKSUM_SEED 0 -#define LZ4S_EOS 0 -#define LZ4S_MAXHEADERSIZE (MAGICNUMBER_SIZE+2+8+4+1) +#define LZ4IO_BLOCKSIZEID_DEFAULT 7 + +#define sizeT sizeof(size_t) +#define maskT (sizeT - 1) /************************************** * Macros -***************************************/ +**************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } -#define DISPLAYUPDATE(l, ...) if (displayLevel>=l) { \ - if ((LZ4IO_GetMilliSpan(g_time) > refreshRate) || (displayLevel>=4)) \ +#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static int g_displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */ + +#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ + if ((LZ4IO_GetMilliSpan(g_time) > refreshRate) || (g_displayLevel>=4)) \ { g_time = clock(); DISPLAY(__VA_ARGS__); \ - if (displayLevel>=4) fflush(stdout); } } + if (g_displayLevel>=4) fflush(stdout); } } static const unsigned refreshRate = 150; static clock_t g_time = 0; /************************************** * Local Parameters -***************************************/ -static int displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */ -static int overwrite = 1; -static int globalBlockSizeId = LZ4S_BLOCKSIZEID_DEFAULT; -static int blockChecksum = 0; -static int streamChecksum = 1; -static int blockIndependence = 1; +**************************************/ +static int g_overwrite = 1; +static int g_blockSizeId = LZ4IO_BLOCKSIZEID_DEFAULT; +static int g_blockChecksum = 0; +static int g_streamChecksum = 1; +static int g_blockIndependence = 1; +static int g_sparseFileSupport = 0; +static int g_contentSizeFlag = 0; static const int minBlockSizeID = 4; static const int maxBlockSizeID = 7; @@ -152,11 +151,10 @@ static const int maxBlockSizeID = 7; /************************************** * Version modifiers -***************************************/ +**************************************/ #define EXTENDED_ARGUMENTS #define EXTENDED_HELP #define EXTENDED_FORMAT -#define DEFAULT_COMPRESSOR compress_file #define DEFAULT_DECOMPRESSOR decodeLZ4S @@ -167,8 +165,8 @@ static const int maxBlockSizeID = 7; /* Default setting : overwrite = 1; return : overwrite mode (0/1) */ int LZ4IO_setOverwrite(int yes) { - overwrite = (yes!=0); - return overwrite; + g_overwrite = (yes!=0); + return g_overwrite; } /* blockSizeID : valid values : 4-5-6-7 */ @@ -176,35 +174,49 @@ int LZ4IO_setBlockSizeID(int bsid) { static const int blockSizeTable[] = { 64 KB, 256 KB, 1 MB, 4 MB }; if ((bsid < minBlockSizeID) || (bsid > maxBlockSizeID)) return -1; - globalBlockSizeId = bsid; - return blockSizeTable[globalBlockSizeId-minBlockSizeID]; + g_blockSizeId = bsid; + return blockSizeTable[g_blockSizeId-minBlockSizeID]; } int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode) { - blockIndependence = (blockMode == LZ4IO_blockIndependent); - return blockIndependence; + g_blockIndependence = (blockMode == LZ4IO_blockIndependent); + return g_blockIndependence; } /* Default setting : no checksum */ int LZ4IO_setBlockChecksumMode(int xxhash) { - blockChecksum = (xxhash != 0); - return blockChecksum; + g_blockChecksum = (xxhash != 0); + return g_blockChecksum; } /* Default setting : checksum enabled */ int LZ4IO_setStreamChecksumMode(int xxhash) { - streamChecksum = (xxhash != 0); - return streamChecksum; + g_streamChecksum = (xxhash != 0); + return g_streamChecksum; } /* Default setting : 0 (no notification) */ int LZ4IO_setNotificationLevel(int level) { - displayLevel = level; - return displayLevel; + g_displayLevel = level; + return g_displayLevel; +} + +/* Default setting : 0 (disabled) */ +int LZ4IO_setSparseFile(int enable) +{ + g_sparseFileSupport = (enable!=0); + return g_sparseFileSupport; +} + +/* Default setting : 0 (disabled) */ +int LZ4IO_setContentSize(int enable) +{ + g_contentSizeFlag = (enable!=0); + return g_contentSizeFlag; } static unsigned LZ4IO_GetMilliSpan(clock_t nPrevious) @@ -214,16 +226,30 @@ static unsigned LZ4IO_GetMilliSpan(clock_t nPrevious) return nSpan; } +static unsigned long long LZ4IO_GetFileSize(const char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); +#else + struct stat statbuf; + r = stat(infilename, &statbuf); +#endif + if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ + return (unsigned long long)statbuf.st_size; +} + -/* ************************************************************************ */ -/* ********************** LZ4 File / Pipe compression ********************* */ -/* ************************************************************************ */ +/* ************************************************************************ ** +** ********************** LZ4 File / Pipe compression ********************* ** +** ************************************************************************ */ -static int LZ4S_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } -static int LZ4S_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4S_SKIPPABLEMASK) == LZ4S_SKIPPABLE0; } +static int LZ4IO_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } +static int LZ4IO_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4IO_SKIPPABLEMASK) == LZ4IO_SKIPPABLE0; } -static int get_fileHandle(char* input_filename, char* output_filename, FILE** pfinput, FILE** pfoutput) +static int get_fileHandle(const char* input_filename, const char* output_filename, FILE** pfinput, FILE** pfoutput) { if (!strcmp (input_filename, stdinmark)) @@ -251,12 +277,12 @@ static int get_fileHandle(char* input_filename, char* output_filename, FILE** pf if (*pfoutput!=0) { fclose(*pfoutput); - if (!overwrite) + if (!g_overwrite) { char ch; DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename); DISPLAYLEVEL(2, "Overwrite ? (Y/N) : "); - if (displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); /* No interaction possible */ + if (g_displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); /* No interaction possible */ ch = (char)getchar(); if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); } @@ -272,15 +298,14 @@ static int get_fileHandle(char* input_filename, char* output_filename, FILE** pf - /*************************************** - * Legacy Compression - * *************************************/ +* Legacy Compression +***************************************/ /* unoptimized version; solves endianess & alignment issues */ static void LZ4IO_writeLE32 (void* p, unsigned value32) { - unsigned char* dstPtr = p; + unsigned char* dstPtr = (unsigned char*)p; dstPtr[0] = (unsigned char)value32; dstPtr[1] = (unsigned char)(value32 >> 8); dstPtr[2] = (unsigned char)(value32 >> 16); @@ -290,7 +315,7 @@ static void LZ4IO_writeLE32 (void* p, unsigned value32) /* LZ4IO_compressFilename_Legacy : * This function is intentionally "hidden" (not published in .h) * It generates compressed streams using the old 'legacy' format */ -int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, int compressionlevel) +int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel) { int (*compressionFunction)(const char*, char*, int); unsigned long long filesize = 0; @@ -308,7 +333,7 @@ int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, i if (compressionlevel < 3) compressionFunction = LZ4_compress; else compressionFunction = LZ4_compressHC; get_fileHandle(input_filename, output_filename, &finput, &foutput); - if ((displayLevel==2) && (compressionlevel==1)) displayLevel=3; + if ((g_displayLevel==2) && (compressionlevel==1)) g_displayLevel=3; /* Allocate Memory */ in_buff = (char*)malloc(LEGACY_BLOCKSIZE); @@ -360,11 +385,11 @@ int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, i } -/*********************************************** - * Compression using Frame format - * ********************************************/ +/********************************************* +* Compression using Frame format +*********************************************/ -int LZ4IO_compressFilename(char* input_filename, char* output_filename, int compressionLevel) +int LZ4IO_compressFilename(const char* input_filename, const char* output_filename, int compressionLevel) { unsigned long long filesize = 0; unsigned long long compressedfilesize = 0; @@ -377,23 +402,29 @@ int LZ4IO_compressFilename(char* input_filename, char* output_filename, int comp size_t sizeCheck, headerSize, readSize, outBuffSize; LZ4F_compressionContext_t ctx; LZ4F_errorCode_t errorCode; - LZ4F_preferences_t prefs = {0}; + LZ4F_preferences_t prefs; /* Init */ start = clock(); - if ((displayLevel==2) && (compressionLevel>=3)) displayLevel=3; + memset(&prefs, 0, sizeof(prefs)); + if ((g_displayLevel==2) && (compressionLevel>=3)) g_displayLevel=3; errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode)); get_fileHandle(input_filename, output_filename, &finput, &foutput); - blockSize = LZ4S_GetBlockSize_FromBlockId (globalBlockSizeId); + blockSize = LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId); /* Set compression parameters */ prefs.autoFlush = 1; prefs.compressionLevel = compressionLevel; - prefs.frameInfo.blockMode = blockIndependence; - prefs.frameInfo.blockSizeID = globalBlockSizeId; - prefs.frameInfo.contentChecksumFlag = streamChecksum; + prefs.frameInfo.blockMode = (blockMode_t)g_blockIndependence; + prefs.frameInfo.blockSizeID = (blockSizeID_t)g_blockSizeId; + prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)g_streamChecksum; + if (g_contentSizeFlag) + { + unsigned long long fileSize = LZ4IO_GetFileSize(input_filename); + prefs.frameInfo.frameOSize = fileSize; /* == 0 if input == stdin */ + } /* Allocate Memory */ in_buff = (char*)malloc(blockSize); @@ -462,13 +493,34 @@ int LZ4IO_compressFilename(char* input_filename, char* output_filename, int comp } +#define FNSPACE 30 +int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel) +{ + int i; + char* outFileName = (char*)malloc(FNSPACE); + size_t ofnSize = FNSPACE; + const size_t suffixSize = strlen(suffix); + + for (i=0; i<ifntSize; i++) + { + size_t ifnSize = strlen(inFileNamesTable[i]); + if (ofnSize <= ifnSize+suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); } + strcpy(outFileName, inFileNamesTable[i]); + strcat(outFileName, suffix); + LZ4IO_compressFilename(inFileNamesTable[i], outFileName, compressionlevel); + } + free(outFileName); + return 0; +} + + /* ********************************************************************* */ -/* ********************** LZ4 File / Stream decoding ******************* */ +/* ********************** LZ4 file-stream Decompression **************** */ /* ********************************************************************* */ static unsigned LZ4IO_readLE32 (const void* s) { - const unsigned char* srcPtr = s; + const unsigned char* srcPtr = (const unsigned char*)s; unsigned value32 = srcPtr[0]; value32 += (srcPtr[1]<<8); value32 += (srcPtr[2]<<16); @@ -506,15 +558,16 @@ static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput) /* Read Block */ sizeCheck = fread(in_buff, 1, blockSize, finput); + if (sizeCheck!=blockSize) EXM_THROW(52, "Read error : cannot access compressed block !"); /* Decode Block */ decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE); - if (decodeSize < 0) EXM_THROW(52, "Decoding Failed ! Corrupted input detected !"); + if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !"); filesize += decodeSize; /* Write Block */ sizeCheck = fwrite(out_buff, 1, decodeSize, foutput); - if (sizeCheck != (size_t)decodeSize) EXM_THROW(53, "Write error : cannot write decoded block into output\n"); + if (sizeCheck != (size_t)decodeSize) EXM_THROW(54, "Write error : cannot write decoded block into output\n"); } /* Free */ @@ -528,57 +581,131 @@ static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput) static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) { unsigned long long filesize = 0; - char* inBuff; - char* outBuff; + void* inBuff; + void* outBuff; # define HEADERMAX 20 char headerBuff[HEADERMAX]; - size_t sizeCheck, nextToRead, outBuffSize, inBuffSize; + size_t sizeCheck; + const size_t inBuffSize = 256 KB; + const size_t outBuffSize = 256 KB; LZ4F_decompressionContext_t ctx; LZ4F_errorCode_t errorCode; - LZ4F_frameInfo_t frameInfo; + unsigned storedSkips = 0; /* init */ errorCode = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) EXM_THROW(60, "Allocation error : can't create context : %s", LZ4F_getErrorName(errorCode)); - LZ4IO_writeLE32(headerBuff, LZ4S_MAGICNUMBER); /* regenerated here, as it was already read from finput */ - - /* Decode stream descriptor */ - outBuffSize = 0; inBuffSize = 0; sizeCheck = MAGICNUMBER_SIZE; - nextToRead = LZ4F_decompress(ctx, NULL, &outBuffSize, headerBuff, &sizeCheck, NULL); - if (LZ4F_isError(nextToRead)) EXM_THROW(61, "Decompression error : %s", LZ4F_getErrorName(nextToRead)); - if (nextToRead > HEADERMAX) EXM_THROW(62, "Header too large (%i>%i)", (int)nextToRead, HEADERMAX); - sizeCheck = fread(headerBuff, 1, nextToRead, finput); - if (sizeCheck!=nextToRead) EXM_THROW(63, "Read error "); - nextToRead = LZ4F_decompress(ctx, NULL, &outBuffSize, headerBuff, &sizeCheck, NULL); - errorCode = LZ4F_getFrameInfo(ctx, &frameInfo, NULL, &inBuffSize); - if (LZ4F_isError(errorCode)) EXM_THROW(64, "can't decode frame header : %s", LZ4F_getErrorName(errorCode)); + if (LZ4F_isError(errorCode)) EXM_THROW(60, "Can't create context : %s", LZ4F_getErrorName(errorCode)); + LZ4IO_writeLE32(headerBuff, LZ4IO_MAGICNUMBER); /* regenerated here, as it was already read from finput */ /* Allocate Memory */ - outBuffSize = LZ4IO_setBlockSizeID(frameInfo.blockSizeID); - inBuffSize = outBuffSize + 4; - inBuff = (char*)malloc(inBuffSize); - outBuff = (char*)malloc(outBuffSize); - if (!inBuff || !outBuff) EXM_THROW(65, "Allocation error : not enough memory"); + inBuff = malloc(256 KB); + outBuff = malloc(256 KB); + if (!inBuff || !outBuff) EXM_THROW(61, "Allocation error : not enough memory"); + + /* Init feed with magic number (already consumed from FILE) */ + { + size_t inSize = 4; + size_t outSize=0; + LZ4IO_writeLE32(inBuff, LZ4IO_MAGICNUMBER); + errorCode = LZ4F_decompress(ctx, outBuff, &outSize, inBuff, &inSize, NULL); + if (LZ4F_isError(errorCode)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(errorCode)); + } + /* Main Loop */ - while (nextToRead != 0) + for (;;) { - size_t decodedBytes = outBuffSize; + size_t readSize; + size_t pos = 0; - /* Read Block */ - sizeCheck = fread(inBuff, 1, nextToRead, finput); - if (sizeCheck!=nextToRead) EXM_THROW(66, "Read error "); + /* Read input */ + readSize = fread(inBuff, 1, inBuffSize, finput); + if (!readSize) break; /* empty file or stream */ - /* Decode Block */ - errorCode = LZ4F_decompress(ctx, outBuff, &decodedBytes, inBuff, &sizeCheck, NULL); - if (LZ4F_isError(errorCode)) EXM_THROW(67, "Decompression error : %s", LZ4F_getErrorName(errorCode)); - if (sizeCheck!=nextToRead) EXM_THROW(67, "Synchronization error"); - nextToRead = errorCode; - filesize += decodedBytes; + while (pos < readSize) + { + /* Decode Input (at least partially) */ + size_t remaining = readSize - pos; + size_t decodedBytes = outBuffSize; + errorCode = LZ4F_decompress(ctx, outBuff, &decodedBytes, (char*)inBuff+pos, &remaining, NULL); + if (LZ4F_isError(errorCode)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(errorCode)); + pos += remaining; + + if (decodedBytes) + { + /* Write Block */ + filesize += decodedBytes; + if (g_sparseFileSupport) + { + size_t* const oBuffStartT = (size_t*)outBuff; /* since outBuff is malloc'ed, it's aligned on size_t */ + size_t* oBuffPosT = oBuffStartT; + size_t oBuffSizeT = decodedBytes / sizeT; + size_t* const oBuffEndT = oBuffStartT + oBuffSizeT; + static const size_t bs0T = (32 KB) / sizeT; + while (oBuffPosT < oBuffEndT) + { + size_t seg0SizeT = bs0T; + size_t nb0T; + int seekResult; + if (seg0SizeT > oBuffSizeT) seg0SizeT = oBuffSizeT; + oBuffSizeT -= seg0SizeT; + for (nb0T=0; (nb0T < seg0SizeT) && (oBuffPosT[nb0T] == 0); nb0T++) ; + storedSkips += (unsigned)(nb0T * sizeT); + if (storedSkips > 1 GB) /* avoid int overflow */ + { + seekResult = fseek(foutput, 1 GB, SEEK_CUR); + if (seekResult != 0) EXM_THROW(68, "1 GB skip error (sparse file)"); + storedSkips -= 1 GB; + } + if (nb0T != seg0SizeT) /* not all 0s */ + { + seekResult = fseek(foutput, storedSkips, SEEK_CUR); + if (seekResult) EXM_THROW(68, "Skip error (sparse file)"); + storedSkips = 0; + seg0SizeT -= nb0T; + oBuffPosT += nb0T; + sizeCheck = fwrite(oBuffPosT, sizeT, seg0SizeT, foutput); + if (sizeCheck != seg0SizeT) EXM_THROW(68, "Write error : cannot write decoded block"); + } + oBuffPosT += seg0SizeT; + } + if (decodedBytes & maskT) /* size not multiple of sizeT (necessarily end of block) */ + { + const char* const restStart = (char*)oBuffEndT; + const char* restPtr = restStart; + size_t restSize = decodedBytes & maskT; + const char* const restEnd = restStart + restSize; + for (; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; + storedSkips += (unsigned) (restPtr - restStart); + if (restPtr != restEnd) + { + int seekResult = fseek(foutput, storedSkips, SEEK_CUR); + if (seekResult) EXM_THROW(68, "Skip error (end of block)"); + storedSkips = 0; + sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, foutput); + if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(68, "Write error : cannot write decoded end of block"); + } + } + } + else + { + sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput); + if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block"); + } + } + } - /* Write Block */ - sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput); - if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block\n"); + } + + if ((g_sparseFileSupport) && (storedSkips>0)) + { + int seekResult; + storedSkips --; + seekResult = fseek(foutput, storedSkips, SEEK_CUR); + if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n"); + memset(outBuff, 0, 1); + sizeCheck = fwrite(outBuff, 1, 1, foutput); + if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n"); } /* Free */ @@ -591,6 +718,28 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) } +static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigned char U32store[MAGICNUMBER_SIZE]) +{ + void* buffer = malloc(64 KB); + size_t read = 1, sizeCheck; + unsigned long long total = MAGICNUMBER_SIZE; + + sizeCheck = fwrite(U32store, 1, MAGICNUMBER_SIZE, foutput); + if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(50, "Pass-through error at start"); + + while (read) + { + read = fread(buffer, 1, 64 KB, finput); + total += read; + sizeCheck = fwrite(buffer, 1, read, foutput); + if (sizeCheck != read) EXM_THROW(50, "Pass-through error"); + } + + free(buffer); + return total; +} + + #define ENDOFSTREAM ((unsigned long long)-1) static unsigned long long selectDecoder( FILE* finput, FILE* foutput) { @@ -598,22 +747,26 @@ static unsigned long long selectDecoder( FILE* finput, FILE* foutput) unsigned magicNumber, size; int errorNb; size_t nbReadBytes; + static unsigned nbCalls = 0; + + /* init */ + nbCalls++; /* Check Archive Header */ nbReadBytes = fread(U32store, 1, MAGICNUMBER_SIZE, finput); if (nbReadBytes==0) return ENDOFSTREAM; /* EOF */ if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(40, "Unrecognized header : Magic Number unreadable"); magicNumber = LZ4IO_readLE32(U32store); /* Little Endian format */ - if (LZ4S_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4S_SKIPPABLE0; /* fold skippable magic numbers */ + if (LZ4IO_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4IO_SKIPPABLE0; /* fold skippable magic numbers */ switch(magicNumber) { - case LZ4S_MAGICNUMBER: + case LZ4IO_MAGICNUMBER: return DEFAULT_DECOMPRESSOR(finput, foutput); case LEGACY_MAGICNUMBER: DISPLAYLEVEL(4, "Detected : Legacy format \n"); return decodeLegacyStream(finput, foutput); - case LZ4S_SKIPPABLE0: + case LZ4IO_SKIPPABLE0: DISPLAYLEVEL(4, "Skipping detected skippable area \n"); nbReadBytes = fread(U32store, 1, 4, finput); if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable"); @@ -623,14 +776,19 @@ static unsigned long long selectDecoder( FILE* finput, FILE* foutput) return selectDecoder(finput, foutput); EXTENDED_FORMAT; default: - if (ftell(finput) == MAGICNUMBER_SIZE) EXM_THROW(44,"Unrecognized header : file cannot be decoded"); /* Wrong magic number at the beginning of 1st stream */ + if (nbCalls == 1) /* just started */ + { + if (g_overwrite) + return LZ4IO_passThrough(finput, foutput, U32store); + EXM_THROW(44,"Unrecognized header : file cannot be decoded"); /* Wrong magic number at the beginning of 1st stream */ + } DISPLAYLEVEL(2, "Stream followed by unrecognized data\n"); return ENDOFSTREAM; } } -int LZ4IO_decompressFilename(char* input_filename, char* output_filename) +int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename) { unsigned long long filesize = 0, decodedSize=0; FILE* finput; @@ -642,6 +800,9 @@ int LZ4IO_decompressFilename(char* input_filename, char* output_filename) start = clock(); get_fileHandle(input_filename, output_filename, &finput, &foutput); + /* sparse file */ + if (g_sparseFileSupport && foutput) { SET_SPARSE_FILE_MODE(foutput); } + /* Loop over multiple streams */ do { |