diff options
Diffstat (limited to 'programs/lz4io.c')
| -rw-r--r-- | programs/lz4io.c | 207 |
1 files changed, 121 insertions, 86 deletions
diff --git a/programs/lz4io.c b/programs/lz4io.c index 8d356c1..f5c5e98 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -46,10 +46,12 @@ /***************************** * Includes *****************************/ -#include <stdio.h> /* fprintf, fopen, fread, stdin, stdout */ -#include <stdlib.h> /* malloc, free */ -#include <string.h> /* strcmp, strlen */ -#include <time.h> /* clock */ +#include <stdio.h> /* fprintf, fopen, fread, stdin, stdout */ +#include <stdlib.h> /* malloc, free */ +#include <string.h> /* strcmp, strlen */ +#include <time.h> /* clock */ +#include <sys/types.h> /* stat64 */ +#include <sys/stat.h> /* stat64 */ #include "lz4io.h" #include "lz4.h" /* still required for legacy format */ #include "lz4hc.h" /* still required for legacy format */ @@ -126,6 +128,7 @@ static int g_blockChecksum = 0; static int g_streamChecksum = 1; static int g_blockIndependence = 1; static int g_sparseFileSupport = 0; +static int g_contentSizeFlag = 0; static const int minBlockSizeID = 4; static const int maxBlockSizeID = 7; @@ -209,6 +212,13 @@ int LZ4IO_setSparseFile(int enable) return g_sparseFileSupport; } +/* Default setting : 0 (disabled) */ +int LZ4IO_setContentSize(int enable) +{ + g_contentSizeFlag = (enable!=0); + return g_contentSizeFlag; +} + static unsigned LZ4IO_GetMilliSpan(clock_t nPrevious) { clock_t nCurrent = clock(); @@ -216,6 +226,20 @@ static unsigned LZ4IO_GetMilliSpan(clock_t nPrevious) return nSpan; } +static unsigned long long LZ4IO_GetFileSize(const char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); +#else + struct stat statbuf; + r = stat(infilename, &statbuf); +#endif + if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ + return (unsigned long long)statbuf.st_size; +} + /* ************************************************************************ ** ** ********************** LZ4 File / Pipe compression ********************* ** @@ -396,6 +420,11 @@ int LZ4IO_compressFilename(const char* input_filename, const char* output_filena prefs.frameInfo.blockMode = (blockMode_t)g_blockIndependence; prefs.frameInfo.blockSizeID = (blockSizeID_t)g_blockSizeId; prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)g_streamChecksum; + if (g_contentSizeFlag) + { + unsigned long long fileSize = LZ4IO_GetFileSize(input_filename); + prefs.frameInfo.frameOSize = fileSize; /* == 0 if input == stdin */ + } /* Allocate Memory */ in_buff = (char*)malloc(blockSize); @@ -485,7 +514,6 @@ int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, } - /* ********************************************************************* */ /* ********************** LZ4 file-stream Decompression **************** */ /* ********************************************************************* */ @@ -557,109 +585,116 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) void* outBuff; # define HEADERMAX 20 char headerBuff[HEADERMAX]; - size_t sizeCheck, nextToRead, outBuffSize, inBuffSize; + size_t sizeCheck; + const size_t inBuffSize = 256 KB; + const size_t outBuffSize = 256 KB; LZ4F_decompressionContext_t ctx; LZ4F_errorCode_t errorCode; - LZ4F_frameInfo_t frameInfo; unsigned storedSkips = 0; /* init */ errorCode = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) EXM_THROW(60, "Allocation error : can't create context : %s", LZ4F_getErrorName(errorCode)); + if (LZ4F_isError(errorCode)) EXM_THROW(60, "Can't create context : %s", LZ4F_getErrorName(errorCode)); LZ4IO_writeLE32(headerBuff, LZ4IO_MAGICNUMBER); /* regenerated here, as it was already read from finput */ - /* Decode stream descriptor */ - outBuffSize = 0; inBuffSize = 0; sizeCheck = MAGICNUMBER_SIZE; - nextToRead = LZ4F_decompress(ctx, NULL, &outBuffSize, headerBuff, &sizeCheck, NULL); - if (LZ4F_isError(nextToRead)) EXM_THROW(61, "Decompression error : %s", LZ4F_getErrorName(nextToRead)); - if (nextToRead > HEADERMAX) EXM_THROW(62, "Header too large (%i>%i)", (int)nextToRead, HEADERMAX); - sizeCheck = fread(headerBuff, 1, nextToRead, finput); - if (sizeCheck!=nextToRead) EXM_THROW(63, "Read error "); - nextToRead = LZ4F_decompress(ctx, NULL, &outBuffSize, headerBuff, &sizeCheck, NULL); - errorCode = LZ4F_getFrameInfo(ctx, &frameInfo, NULL, &inBuffSize); - if (LZ4F_isError(errorCode)) EXM_THROW(64, "can't decode frame header : %s", LZ4F_getErrorName(errorCode)); - /* Allocate Memory */ - outBuffSize = LZ4IO_setBlockSizeID(frameInfo.blockSizeID); - inBuffSize = outBuffSize + 4; - inBuff = malloc(inBuffSize); - outBuff = malloc(outBuffSize); - if (!inBuff || !outBuff) EXM_THROW(65, "Allocation error : not enough memory"); + inBuff = malloc(256 KB); + outBuff = malloc(256 KB); + if (!inBuff || !outBuff) EXM_THROW(61, "Allocation error : not enough memory"); - /* Main Loop */ - while (nextToRead != 0) + /* Init feed with magic number (already consumed from FILE) */ { - size_t decodedBytes = outBuffSize; + size_t inSize = 4; + size_t outSize=0; + LZ4IO_writeLE32(inBuff, LZ4IO_MAGICNUMBER); + errorCode = LZ4F_decompress(ctx, outBuff, &outSize, inBuff, &inSize, NULL); + if (LZ4F_isError(errorCode)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(errorCode)); + } - /* Read Block */ - sizeCheck = fread(inBuff, 1, nextToRead, finput); - if (sizeCheck!=nextToRead) EXM_THROW(66, "Read error "); - /* Decode Block */ - errorCode = LZ4F_decompress(ctx, outBuff, &decodedBytes, inBuff, &sizeCheck, NULL); - if (LZ4F_isError(errorCode)) EXM_THROW(67, "Decompression error : %s", LZ4F_getErrorName(errorCode)); - if (sizeCheck!=nextToRead) EXM_THROW(67, "Synchronization error"); - nextToRead = errorCode; - filesize += decodedBytes; + /* Main Loop */ + for (;;) + { + size_t readSize; + size_t pos = 0; - /* Write Block */ - if (g_sparseFileSupport) + /* Read input */ + readSize = fread(inBuff, 1, inBuffSize, finput); + if (!readSize) break; /* empty file or stream */ + + while (pos < readSize) { - size_t* const oBuffStartT = (size_t*)outBuff; /* since outBuff is malloc'ed, it's aligned on size_t */ - size_t* oBuffPosT = oBuffStartT; - size_t oBuffSizeT = decodedBytes / sizeT; - size_t* const oBuffEndT = oBuffStartT + oBuffSizeT; - static const size_t bs0T = (32 KB) / sizeT; - while (oBuffPosT < oBuffEndT) + /* Decode Input (at least partially) */ + size_t remaining = readSize - pos; + size_t decodedBytes = outBuffSize; + errorCode = LZ4F_decompress(ctx, outBuff, &decodedBytes, (char*)inBuff+pos, &remaining, NULL); + if (LZ4F_isError(errorCode)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(errorCode)); + pos += remaining; + + if (decodedBytes) { - size_t seg0SizeT = bs0T; - size_t nb0T; - int seekResult; - if (seg0SizeT > oBuffSizeT) seg0SizeT = oBuffSizeT; - oBuffSizeT -= seg0SizeT; - for (nb0T=0; (nb0T < seg0SizeT) && (oBuffPosT[nb0T] == 0); nb0T++) ; - storedSkips += (unsigned)(nb0T * sizeT); - if (storedSkips > 1 GB) /* avoid int overflow */ + /* Write Block */ + filesize += decodedBytes; + if (g_sparseFileSupport) { - seekResult = fseek(foutput, 1 GB, SEEK_CUR); - if (seekResult != 0) EXM_THROW(68, "1 GB skip error (sparse file)"); - storedSkips -= 1 GB; + size_t* const oBuffStartT = (size_t*)outBuff; /* since outBuff is malloc'ed, it's aligned on size_t */ + size_t* oBuffPosT = oBuffStartT; + size_t oBuffSizeT = decodedBytes / sizeT; + size_t* const oBuffEndT = oBuffStartT + oBuffSizeT; + static const size_t bs0T = (32 KB) / sizeT; + while (oBuffPosT < oBuffEndT) + { + size_t seg0SizeT = bs0T; + size_t nb0T; + int seekResult; + if (seg0SizeT > oBuffSizeT) seg0SizeT = oBuffSizeT; + oBuffSizeT -= seg0SizeT; + for (nb0T=0; (nb0T < seg0SizeT) && (oBuffPosT[nb0T] == 0); nb0T++) ; + storedSkips += (unsigned)(nb0T * sizeT); + if (storedSkips > 1 GB) /* avoid int overflow */ + { + seekResult = fseek(foutput, 1 GB, SEEK_CUR); + if (seekResult != 0) EXM_THROW(68, "1 GB skip error (sparse file)"); + storedSkips -= 1 GB; + } + if (nb0T != seg0SizeT) /* not all 0s */ + { + seekResult = fseek(foutput, storedSkips, SEEK_CUR); + if (seekResult) EXM_THROW(68, "Skip error (sparse file)"); + storedSkips = 0; + seg0SizeT -= nb0T; + oBuffPosT += nb0T; + sizeCheck = fwrite(oBuffPosT, sizeT, seg0SizeT, foutput); + if (sizeCheck != seg0SizeT) EXM_THROW(68, "Write error : cannot write decoded block"); + } + oBuffPosT += seg0SizeT; + } + if (decodedBytes & maskT) /* size not multiple of sizeT (necessarily end of block) */ + { + const char* const restStart = (char*)oBuffEndT; + const char* restPtr = restStart; + size_t restSize = decodedBytes & maskT; + const char* const restEnd = restStart + restSize; + for (; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; + storedSkips += (unsigned) (restPtr - restStart); + if (restPtr != restEnd) + { + int seekResult = fseek(foutput, storedSkips, SEEK_CUR); + if (seekResult) EXM_THROW(68, "Skip error (end of block)"); + storedSkips = 0; + sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, foutput); + if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(68, "Write error : cannot write decoded end of block"); + } + } } - if (nb0T != seg0SizeT) /* not all 0s */ + else { - seekResult = fseek(foutput, storedSkips, SEEK_CUR); - if (seekResult) EXM_THROW(68, "Skip error (sparse file)"); - storedSkips = 0; - seg0SizeT -= nb0T; - oBuffPosT += nb0T; - sizeCheck = fwrite(oBuffPosT, sizeT, seg0SizeT, foutput); - if (sizeCheck != seg0SizeT) EXM_THROW(68, "Write error : cannot write decoded block"); - } - oBuffPosT += seg0SizeT; - } - if (decodedBytes & maskT) /* size not multiple of sizeT (necessarily end of block) */ - { - const char* const restStart = (char*)oBuffEndT; - const char* restPtr = restStart; - size_t restSize = decodedBytes & maskT; - const char* const restEnd = restStart + restSize; - for (; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; - storedSkips += (unsigned) (restPtr - restStart); - if (restPtr != restEnd) - { - int seekResult = fseek(foutput, storedSkips, SEEK_CUR); - if (seekResult) EXM_THROW(68, "Skip error (end of block)"); - storedSkips = 0; - sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, foutput); - if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(68, "Write error : cannot write decoded end of block"); + sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput); + if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block"); } } } - else - { - sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput); - if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block"); - } + } if ((g_sparseFileSupport) && (storedSkips>0)) |