summaryrefslogtreecommitdiffstats
path: root/programs/lz4io.c
diff options
context:
space:
mode:
authorYann Collet <yann.collet.73@gmail.com>2015-03-23 01:20:42 (GMT)
committerYann Collet <yann.collet.73@gmail.com>2015-03-23 01:20:42 (GMT)
commit7f436a1215f11b0fb872c34f088b8b5888d0630d (patch)
tree51466bdcec0166a2e25d4dc276abac6f284d7f72 /programs/lz4io.c
parenta28b14731d35f717c5d48f07c9a9312c60961493 (diff)
downloadlz4-7f436a1215f11b0fb872c34f088b8b5888d0630d.zip
lz4-7f436a1215f11b0fb872c34f088b8b5888d0630d.tar.gz
lz4-7f436a1215f11b0fb872c34f088b8b5888d0630d.tar.bz2
lz4 cli supports frame content size
Diffstat (limited to 'programs/lz4io.c')
-rw-r--r--programs/lz4io.c207
1 files changed, 121 insertions, 86 deletions
diff --git a/programs/lz4io.c b/programs/lz4io.c
index 8d356c1..f5c5e98 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -46,10 +46,12 @@
/*****************************
* Includes
*****************************/
-#include <stdio.h> /* fprintf, fopen, fread, stdin, stdout */
-#include <stdlib.h> /* malloc, free */
-#include <string.h> /* strcmp, strlen */
-#include <time.h> /* clock */
+#include <stdio.h> /* fprintf, fopen, fread, stdin, stdout */
+#include <stdlib.h> /* malloc, free */
+#include <string.h> /* strcmp, strlen */
+#include <time.h> /* clock */
+#include <sys/types.h> /* stat64 */
+#include <sys/stat.h> /* stat64 */
#include "lz4io.h"
#include "lz4.h" /* still required for legacy format */
#include "lz4hc.h" /* still required for legacy format */
@@ -126,6 +128,7 @@ static int g_blockChecksum = 0;
static int g_streamChecksum = 1;
static int g_blockIndependence = 1;
static int g_sparseFileSupport = 0;
+static int g_contentSizeFlag = 0;
static const int minBlockSizeID = 4;
static const int maxBlockSizeID = 7;
@@ -209,6 +212,13 @@ int LZ4IO_setSparseFile(int enable)
return g_sparseFileSupport;
}
+/* Default setting : 0 (disabled) */
+int LZ4IO_setContentSize(int enable)
+{
+ g_contentSizeFlag = (enable!=0);
+ return g_contentSizeFlag;
+}
+
static unsigned LZ4IO_GetMilliSpan(clock_t nPrevious)
{
clock_t nCurrent = clock();
@@ -216,6 +226,20 @@ static unsigned LZ4IO_GetMilliSpan(clock_t nPrevious)
return nSpan;
}
+static unsigned long long LZ4IO_GetFileSize(const char* infilename)
+{
+ int r;
+#if defined(_MSC_VER)
+ struct _stat64 statbuf;
+ r = _stat64(infilename, &statbuf);
+#else
+ struct stat statbuf;
+ r = stat(infilename, &statbuf);
+#endif
+ if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
+ return (unsigned long long)statbuf.st_size;
+}
+
/* ************************************************************************ **
** ********************** LZ4 File / Pipe compression ********************* **
@@ -396,6 +420,11 @@ int LZ4IO_compressFilename(const char* input_filename, const char* output_filena
prefs.frameInfo.blockMode = (blockMode_t)g_blockIndependence;
prefs.frameInfo.blockSizeID = (blockSizeID_t)g_blockSizeId;
prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)g_streamChecksum;
+ if (g_contentSizeFlag)
+ {
+ unsigned long long fileSize = LZ4IO_GetFileSize(input_filename);
+ prefs.frameInfo.frameOSize = fileSize; /* == 0 if input == stdin */
+ }
/* Allocate Memory */
in_buff = (char*)malloc(blockSize);
@@ -485,7 +514,6 @@ int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize,
}
-
/* ********************************************************************* */
/* ********************** LZ4 file-stream Decompression **************** */
/* ********************************************************************* */
@@ -557,109 +585,116 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
void* outBuff;
# define HEADERMAX 20
char headerBuff[HEADERMAX];
- size_t sizeCheck, nextToRead, outBuffSize, inBuffSize;
+ size_t sizeCheck;
+ const size_t inBuffSize = 256 KB;
+ const size_t outBuffSize = 256 KB;
LZ4F_decompressionContext_t ctx;
LZ4F_errorCode_t errorCode;
- LZ4F_frameInfo_t frameInfo;
unsigned storedSkips = 0;
/* init */
errorCode = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION);
- if (LZ4F_isError(errorCode)) EXM_THROW(60, "Allocation error : can't create context : %s", LZ4F_getErrorName(errorCode));
+ if (LZ4F_isError(errorCode)) EXM_THROW(60, "Can't create context : %s", LZ4F_getErrorName(errorCode));
LZ4IO_writeLE32(headerBuff, LZ4IO_MAGICNUMBER); /* regenerated here, as it was already read from finput */
- /* Decode stream descriptor */
- outBuffSize = 0; inBuffSize = 0; sizeCheck = MAGICNUMBER_SIZE;
- nextToRead = LZ4F_decompress(ctx, NULL, &outBuffSize, headerBuff, &sizeCheck, NULL);
- if (LZ4F_isError(nextToRead)) EXM_THROW(61, "Decompression error : %s", LZ4F_getErrorName(nextToRead));
- if (nextToRead > HEADERMAX) EXM_THROW(62, "Header too large (%i>%i)", (int)nextToRead, HEADERMAX);
- sizeCheck = fread(headerBuff, 1, nextToRead, finput);
- if (sizeCheck!=nextToRead) EXM_THROW(63, "Read error ");
- nextToRead = LZ4F_decompress(ctx, NULL, &outBuffSize, headerBuff, &sizeCheck, NULL);
- errorCode = LZ4F_getFrameInfo(ctx, &frameInfo, NULL, &inBuffSize);
- if (LZ4F_isError(errorCode)) EXM_THROW(64, "can't decode frame header : %s", LZ4F_getErrorName(errorCode));
-
/* Allocate Memory */
- outBuffSize = LZ4IO_setBlockSizeID(frameInfo.blockSizeID);
- inBuffSize = outBuffSize + 4;
- inBuff = malloc(inBuffSize);
- outBuff = malloc(outBuffSize);
- if (!inBuff || !outBuff) EXM_THROW(65, "Allocation error : not enough memory");
+ inBuff = malloc(256 KB);
+ outBuff = malloc(256 KB);
+ if (!inBuff || !outBuff) EXM_THROW(61, "Allocation error : not enough memory");
- /* Main Loop */
- while (nextToRead != 0)
+ /* Init feed with magic number (already consumed from FILE) */
{
- size_t decodedBytes = outBuffSize;
+ size_t inSize = 4;
+ size_t outSize=0;
+ LZ4IO_writeLE32(inBuff, LZ4IO_MAGICNUMBER);
+ errorCode = LZ4F_decompress(ctx, outBuff, &outSize, inBuff, &inSize, NULL);
+ if (LZ4F_isError(errorCode)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(errorCode));
+ }
- /* Read Block */
- sizeCheck = fread(inBuff, 1, nextToRead, finput);
- if (sizeCheck!=nextToRead) EXM_THROW(66, "Read error ");
- /* Decode Block */
- errorCode = LZ4F_decompress(ctx, outBuff, &decodedBytes, inBuff, &sizeCheck, NULL);
- if (LZ4F_isError(errorCode)) EXM_THROW(67, "Decompression error : %s", LZ4F_getErrorName(errorCode));
- if (sizeCheck!=nextToRead) EXM_THROW(67, "Synchronization error");
- nextToRead = errorCode;
- filesize += decodedBytes;
+ /* Main Loop */
+ for (;;)
+ {
+ size_t readSize;
+ size_t pos = 0;
- /* Write Block */
- if (g_sparseFileSupport)
+ /* Read input */
+ readSize = fread(inBuff, 1, inBuffSize, finput);
+ if (!readSize) break; /* empty file or stream */
+
+ while (pos < readSize)
{
- size_t* const oBuffStartT = (size_t*)outBuff; /* since outBuff is malloc'ed, it's aligned on size_t */
- size_t* oBuffPosT = oBuffStartT;
- size_t oBuffSizeT = decodedBytes / sizeT;
- size_t* const oBuffEndT = oBuffStartT + oBuffSizeT;
- static const size_t bs0T = (32 KB) / sizeT;
- while (oBuffPosT < oBuffEndT)
+ /* Decode Input (at least partially) */
+ size_t remaining = readSize - pos;
+ size_t decodedBytes = outBuffSize;
+ errorCode = LZ4F_decompress(ctx, outBuff, &decodedBytes, (char*)inBuff+pos, &remaining, NULL);
+ if (LZ4F_isError(errorCode)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(errorCode));
+ pos += remaining;
+
+ if (decodedBytes)
{
- size_t seg0SizeT = bs0T;
- size_t nb0T;
- int seekResult;
- if (seg0SizeT > oBuffSizeT) seg0SizeT = oBuffSizeT;
- oBuffSizeT -= seg0SizeT;
- for (nb0T=0; (nb0T < seg0SizeT) && (oBuffPosT[nb0T] == 0); nb0T++) ;
- storedSkips += (unsigned)(nb0T * sizeT);
- if (storedSkips > 1 GB) /* avoid int overflow */
+ /* Write Block */
+ filesize += decodedBytes;
+ if (g_sparseFileSupport)
{
- seekResult = fseek(foutput, 1 GB, SEEK_CUR);
- if (seekResult != 0) EXM_THROW(68, "1 GB skip error (sparse file)");
- storedSkips -= 1 GB;
+ size_t* const oBuffStartT = (size_t*)outBuff; /* since outBuff is malloc'ed, it's aligned on size_t */
+ size_t* oBuffPosT = oBuffStartT;
+ size_t oBuffSizeT = decodedBytes / sizeT;
+ size_t* const oBuffEndT = oBuffStartT + oBuffSizeT;
+ static const size_t bs0T = (32 KB) / sizeT;
+ while (oBuffPosT < oBuffEndT)
+ {
+ size_t seg0SizeT = bs0T;
+ size_t nb0T;
+ int seekResult;
+ if (seg0SizeT > oBuffSizeT) seg0SizeT = oBuffSizeT;
+ oBuffSizeT -= seg0SizeT;
+ for (nb0T=0; (nb0T < seg0SizeT) && (oBuffPosT[nb0T] == 0); nb0T++) ;
+ storedSkips += (unsigned)(nb0T * sizeT);
+ if (storedSkips > 1 GB) /* avoid int overflow */
+ {
+ seekResult = fseek(foutput, 1 GB, SEEK_CUR);
+ if (seekResult != 0) EXM_THROW(68, "1 GB skip error (sparse file)");
+ storedSkips -= 1 GB;
+ }
+ if (nb0T != seg0SizeT) /* not all 0s */
+ {
+ seekResult = fseek(foutput, storedSkips, SEEK_CUR);
+ if (seekResult) EXM_THROW(68, "Skip error (sparse file)");
+ storedSkips = 0;
+ seg0SizeT -= nb0T;
+ oBuffPosT += nb0T;
+ sizeCheck = fwrite(oBuffPosT, sizeT, seg0SizeT, foutput);
+ if (sizeCheck != seg0SizeT) EXM_THROW(68, "Write error : cannot write decoded block");
+ }
+ oBuffPosT += seg0SizeT;
+ }
+ if (decodedBytes & maskT) /* size not multiple of sizeT (necessarily end of block) */
+ {
+ const char* const restStart = (char*)oBuffEndT;
+ const char* restPtr = restStart;
+ size_t restSize = decodedBytes & maskT;
+ const char* const restEnd = restStart + restSize;
+ for (; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ;
+ storedSkips += (unsigned) (restPtr - restStart);
+ if (restPtr != restEnd)
+ {
+ int seekResult = fseek(foutput, storedSkips, SEEK_CUR);
+ if (seekResult) EXM_THROW(68, "Skip error (end of block)");
+ storedSkips = 0;
+ sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, foutput);
+ if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(68, "Write error : cannot write decoded end of block");
+ }
+ }
}
- if (nb0T != seg0SizeT) /* not all 0s */
+ else
{
- seekResult = fseek(foutput, storedSkips, SEEK_CUR);
- if (seekResult) EXM_THROW(68, "Skip error (sparse file)");
- storedSkips = 0;
- seg0SizeT -= nb0T;
- oBuffPosT += nb0T;
- sizeCheck = fwrite(oBuffPosT, sizeT, seg0SizeT, foutput);
- if (sizeCheck != seg0SizeT) EXM_THROW(68, "Write error : cannot write decoded block");
- }
- oBuffPosT += seg0SizeT;
- }
- if (decodedBytes & maskT) /* size not multiple of sizeT (necessarily end of block) */
- {
- const char* const restStart = (char*)oBuffEndT;
- const char* restPtr = restStart;
- size_t restSize = decodedBytes & maskT;
- const char* const restEnd = restStart + restSize;
- for (; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ;
- storedSkips += (unsigned) (restPtr - restStart);
- if (restPtr != restEnd)
- {
- int seekResult = fseek(foutput, storedSkips, SEEK_CUR);
- if (seekResult) EXM_THROW(68, "Skip error (end of block)");
- storedSkips = 0;
- sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, foutput);
- if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(68, "Write error : cannot write decoded end of block");
+ sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput);
+ if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block");
}
}
}
- else
- {
- sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput);
- if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block");
- }
+
}
if ((g_sparseFileSupport) && (storedSkips>0))