summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorYann Collet <cyan@fb.com>2020-11-14 22:26:48 (GMT)
committerYann Collet <cyan@fb.com>2020-11-14 22:26:48 (GMT)
commit468011c346a1e73eede3e70ee8009f812b02cc76 (patch)
tree089a982166220e58d116dd2c969c32916c1994cd
parent581c459b4ec4bbaae518a0ef9baa552f21f9e24e (diff)
parent26bc4472c3c9aabeffa9eb6e0762acec6cc7a251 (diff)
downloadlz4-468011c346a1e73eede3e70ee8009f812b02cc76.zip
lz4-468011c346a1e73eede3e70ee8009f812b02cc76.tar.gz
lz4-468011c346a1e73eede3e70ee8009f812b02cc76.tar.bz2
Merge branch 'dev' of github.com:Cyan4973/lz4 into dev
-rw-r--r--lib/lz4frame.h5
-rw-r--r--programs/lz4io.c383
-rw-r--r--tests/Makefile2
3 files changed, 205 insertions, 185 deletions
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 6e91e2d..4573317 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -108,7 +108,7 @@ LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return
/*-************************************
* Frame compression types
- **************************************/
+ ************************************* */
/* #define LZ4F_ENABLE_OBSOLETE_ENUMS // uncomment to enable obsolete enums */
#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS
# define LZ4F_OBSOLETE_ENUM(x) , LZ4F_DEPRECATE(x) = LZ4F_##x
@@ -118,7 +118,8 @@ LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return
/* The larger the block size, the (slightly) better the compression ratio,
* though there are diminishing returns.
- * Larger blocks also increase memory usage on both compression and decompression sides. */
+ * Larger blocks also increase memory usage on both compression and decompression sides.
+ */
typedef enum {
LZ4F_default=0,
LZ4F_max64KB=4,
diff --git a/programs/lz4io.c b/programs/lz4io.c
index 3ffc519..975f590 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -111,20 +111,20 @@ static clock_t g_time = 0;
**************************************/
struct LZ4IO_prefs_s {
- int passThrough;
- int overwrite;
- int testMode;
- int blockSizeId;
- size_t blockSize;
- int blockChecksum;
- int streamChecksum;
- int blockIndependence;
- int sparseFileSupport;
- int contentSizeFlag;
- int useDictionary;
- unsigned favorDecSpeed;
- const char* dictionaryFilename;
- int removeSrcFile;
+ int passThrough;
+ int overwrite;
+ int testMode;
+ int blockSizeId;
+ size_t blockSize;
+ int blockChecksum;
+ int streamChecksum;
+ int blockIndependence;
+ int sparseFileSupport;
+ int contentSizeFlag;
+ int useDictionary;
+ unsigned favorDecSpeed;
+ const char* dictionaryFilename;
+ int removeSrcFile;
};
/**************************************
@@ -325,6 +325,7 @@ static FILE* LZ4IO_openSrcFile(const char* srcFileName)
}
/** FIO_openDstFile() :
+ * prefs is writable, because sparseFileSupport might be updated.
* condition : `dstFileName` must be non-NULL.
* @result : FILE* to `dstFileName`, or NULL if it fails */
static FILE* LZ4IO_openDstFile(LZ4IO_prefs_t* const prefs, const char* dstFileName)
@@ -333,18 +334,19 @@ static FILE* LZ4IO_openDstFile(LZ4IO_prefs_t* const prefs, const char* dstFileNa
assert(dstFileName != NULL);
if (!strcmp (dstFileName, stdoutmark)) {
- DISPLAYLEVEL(4,"Using stdout for output\n");
+ DISPLAYLEVEL(4, "Using stdout for output \n");
f = stdout;
SET_BINARY_MODE(stdout);
if (prefs->sparseFileSupport==1) {
prefs->sparseFileSupport = 0;
- DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n");
+ DISPLAYLEVEL(4, "Sparse File Support automatically disabled on stdout ;"
+ " to force-enable it, add --sparse command \n");
}
} else {
if (!prefs->overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */
- f = fopen( dstFileName, "rb" );
- if (f != NULL) { /* dest exists, prompt for overwrite authorization */
- fclose(f);
+ FILE* const testf = fopen( dstFileName, "rb" );
+ if (testf != NULL) { /* dest exists, prompt for overwrite authorization */
+ fclose(testf);
if (g_displayLevel <= 1) { /* No interaction possible */
DISPLAY("%s already exists; not overwritten \n", dstFileName);
return NULL;
@@ -392,7 +394,9 @@ static int LZ4IO_LZ4_compress(const char* src, char* dst, int srcSize, int dstSi
/* LZ4IO_compressFilename_Legacy :
* This function is intentionally "hidden" (not published in .h)
* It generates compressed streams using the old 'legacy' format */
-int LZ4IO_compressFilename_Legacy(LZ4IO_prefs_t* const prefs, const char* input_filename, const char* output_filename, int compressionlevel)
+int LZ4IO_compressFilename_Legacy(LZ4IO_prefs_t* const prefs,
+ const char* input_filename, const char* output_filename,
+ int compressionlevel)
{
typedef int (*compress_f)(const char* src, char* dst, int srcSize, int dstSize, int cLevel);
compress_f const compressionFunction = (compressionlevel < 3) ? LZ4IO_LZ4_compress : LZ4_compress_HC;
@@ -424,23 +428,22 @@ int LZ4IO_compressFilename_Legacy(LZ4IO_prefs_t* const prefs, const char* input_
/* Write Archive Header */
LZ4IO_writeLE32(out_buff, LEGACY_MAGICNUMBER);
- { size_t const writeSize = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput);
- if (writeSize != MAGICNUMBER_SIZE)
- EXM_THROW(22, "Write error : cannot write header");
- }
+ if (fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput) != MAGICNUMBER_SIZE)
+ EXM_THROW(22, "Write error : cannot write header");
/* Main Loop */
while (1) {
int outSize;
/* Read Block */
size_t const inSize = fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput);
- assert(inSize <= LEGACY_BLOCKSIZE);
if (inSize == 0) break;
+ assert(inSize <= LEGACY_BLOCKSIZE);
filesize += inSize;
/* Compress Block */
outSize = compressionFunction(in_buff, out_buff+4, (int)inSize, outBuffSize, compressionlevel);
- compressedfilesize += outSize+4;
+ assert(outSize >= 0);
+ compressedfilesize += (unsigned long long)outSize+4;
DISPLAYUPDATE(2, "\rRead : %i MB ==> %.2f%% ",
(int)(filesize>>20), (double)compressedfilesize/filesize*100);
@@ -448,9 +451,8 @@ int LZ4IO_compressFilename_Legacy(LZ4IO_prefs_t* const prefs, const char* input_
assert(outSize > 0);
assert(outSize < outBuffSize);
LZ4IO_writeLE32(out_buff, (unsigned)outSize);
- { size_t const writeSize = fwrite(out_buff, 1, outSize+4, foutput);
- if (writeSize != (size_t)(outSize+4))
- EXM_THROW(24, "Write error : cannot write compressed block");
+ if (fwrite(out_buff, 1, (size_t)outSize+4, foutput) != (size_t)(outSize+4)) {
+ EXM_THROW(24, "Write error : cannot write compressed block");
} }
if (ferror(finput)) EXM_THROW(25, "Error while reading %s ", input_filename);
@@ -537,22 +539,20 @@ typedef struct {
LZ4F_CDict* cdict;
} cRess_t;
-static void* LZ4IO_createDict(LZ4IO_prefs_t* const prefs, size_t *dictSize) {
+static void* LZ4IO_createDict(size_t* dictSize, const char* const dictFilename)
+{
size_t readSize;
size_t dictEnd = 0;
size_t dictLen = 0;
size_t dictStart;
size_t circularBufSize = LZ4_MAX_DICT_SIZE;
- char* circularBuf;
- char* dictBuf;
- const char* dictFilename = prefs->dictionaryFilename;
+ char* circularBuf = (char*)malloc(circularBufSize);
+ char* dictBuf;
FILE* dictFile;
+ if (!circularBuf) EXM_THROW(25, "Allocation error : not enough memory for circular buffer");
if (!dictFilename) EXM_THROW(25, "Dictionary error : no filename provided");
- circularBuf = (char *) malloc(circularBufSize);
- if (!circularBuf) EXM_THROW(25, "Allocation error : not enough memory");
-
dictFile = LZ4IO_openSrcFile(dictFilename);
if (!dictFile) EXM_THROW(25, "Dictionary error : could not open dictionary file");
@@ -582,7 +582,7 @@ static void* LZ4IO_createDict(LZ4IO_prefs_t* const prefs, size_t *dictSize) {
circularBuf = NULL;
} else {
/* Otherwise, we will alloc a new buffer and copy our dict into that. */
- dictBuf = (char *) malloc(dictLen ? dictLen : 1);
+ dictBuf = (char *)malloc(dictLen ? dictLen : 1);
if (!dictBuf) EXM_THROW(25, "Allocation error : not enough memory");
memcpy(dictBuf, circularBuf + dictStart, circularBufSize - dictStart);
@@ -595,14 +595,13 @@ static void* LZ4IO_createDict(LZ4IO_prefs_t* const prefs, size_t *dictSize) {
return dictBuf;
}
-static LZ4F_CDict* LZ4IO_createCDict(LZ4IO_prefs_t* const prefs) {
+static LZ4F_CDict* LZ4IO_createCDict(const LZ4IO_prefs_t* const prefs)
+{
size_t dictionarySize;
void* dictionaryBuffer;
LZ4F_CDict* cdict;
- if (!prefs->useDictionary) {
- return NULL;
- }
- dictionaryBuffer = LZ4IO_createDict(prefs, &dictionarySize);
+ if (!prefs->useDictionary) return NULL;
+ dictionaryBuffer = LZ4IO_createDict(&dictionarySize, prefs->dictionaryFilename);
if (!dictionaryBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary");
cdict = LZ4F_createCDict(dictionaryBuffer, dictionarySize);
free(dictionaryBuffer);
@@ -643,6 +642,7 @@ static void LZ4IO_freeCResources(cRess_t ress)
/*
* LZ4IO_compressFilename_extRess()
+ * io_prefs is mutable, as it may update sparseFileSupport
* result : 0 : compression completed correctly
* 1 : missing or pb opening srcFileName
*/
@@ -653,7 +653,6 @@ LZ4IO_compressFilename_extRess(LZ4IO_prefs_t* const io_prefs, cRess_t ress,
{
unsigned long long filesize = 0;
unsigned long long compressedfilesize = 0;
- FILE* srcFile;
FILE* dstFile;
void* const srcBuffer = ress.srcBuffer;
void* const dstBuffer = ress.dstBuffer;
@@ -664,13 +663,12 @@ LZ4IO_compressFilename_extRess(LZ4IO_prefs_t* const io_prefs, cRess_t ress,
LZ4F_preferences_t prefs;
/* Init */
- srcFile = LZ4IO_openSrcFile(srcFileName);
+ FILE* const srcFile = LZ4IO_openSrcFile(srcFileName);
if (srcFile == NULL) return 1;
dstFile = LZ4IO_openDstFile(io_prefs, dstFileName);
if (dstFile == NULL) { fclose(srcFile); return 1; }
memset(&prefs, 0, sizeof(prefs));
-
/* Set compression parameters */
prefs.autoFlush = 1;
prefs.compressionLevel = compressionLevel;
@@ -694,41 +692,41 @@ LZ4IO_compressFilename_extRess(LZ4IO_prefs_t* const io_prefs, cRess_t ress,
/* single-block file */
if (readSize < blockSize) {
/* Compress in single pass */
- size_t cSize = LZ4F_compressFrame_usingCDict(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs);
- if (LZ4F_isError(cSize)) EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize));
+ size_t const cSize = LZ4F_compressFrame_usingCDict(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs);
+ if (LZ4F_isError(cSize))
+ EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize));
compressedfilesize = cSize;
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
(unsigned)(filesize>>20), (double)compressedfilesize/(filesize+!filesize)*100); /* avoid division by zero */
/* Write Block */
- { size_t const sizeCheck = fwrite(dstBuffer, 1, cSize, dstFile);
- if (sizeCheck!=cSize) EXM_THROW(32, "Write error : cannot write compressed block");
+ if (fwrite(dstBuffer, 1, cSize, dstFile) != cSize) {
+ EXM_THROW(32, "Write error : failed writing single-block compressed frame");
} }
else
/* multiple-blocks file */
{
- /* Write Archive Header */
- size_t headerSize = LZ4F_compressBegin_usingCDict(ctx, dstBuffer, dstBufferSize, ress.cdict, &prefs);
+ /* Write Frame Header */
+ size_t const headerSize = LZ4F_compressBegin_usingCDict(ctx, dstBuffer, dstBufferSize, ress.cdict, &prefs);
if (LZ4F_isError(headerSize)) EXM_THROW(33, "File header generation failed : %s", LZ4F_getErrorName(headerSize));
- { size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile);
- if (sizeCheck!=headerSize) EXM_THROW(34, "Write error : cannot write header"); }
+ if (fwrite(dstBuffer, 1, headerSize, dstFile) != headerSize)
+ EXM_THROW(34, "Write error : cannot write header");
compressedfilesize += headerSize;
- /* Main Loop */
+ /* Main Loop - one block at a time */
while (readSize>0) {
- size_t outSize;
-
- /* Compress Block */
- outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL);
- if (LZ4F_isError(outSize)) EXM_THROW(35, "Compression failed : %s", LZ4F_getErrorName(outSize));
+ size_t const outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL);
+ if (LZ4F_isError(outSize))
+ EXM_THROW(35, "Compression failed : %s", LZ4F_getErrorName(outSize));
compressedfilesize += outSize;
- DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100);
+ DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
+ (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100);
/* Write Block */
- { size_t const sizeCheck = fwrite(dstBuffer, 1, outSize, dstFile);
- if (sizeCheck!=outSize) EXM_THROW(36, "Write error : cannot write compressed block"); }
+ if (fwrite(dstBuffer, 1, outSize, dstFile) != outSize)
+ EXM_THROW(36, "Write error : cannot write compressed block");
/* Read next block */
readSize = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile);
@@ -736,18 +734,18 @@ LZ4IO_compressFilename_extRess(LZ4IO_prefs_t* const io_prefs, cRess_t ress,
}
if (ferror(srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName);
- /* End of Stream mark */
- headerSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL);
- if (LZ4F_isError(headerSize)) EXM_THROW(38, "End of file generation failed : %s", LZ4F_getErrorName(headerSize));
-
- { size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile);
- if (sizeCheck!=headerSize) EXM_THROW(39, "Write error : cannot write end of stream"); }
- compressedfilesize += headerSize;
- }
+ /* End of Frame mark */
+ { size_t const endSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL);
+ if (LZ4F_isError(endSize))
+ EXM_THROW(38, "End of frame error : %s", LZ4F_getErrorName(endSize));
+ if (fwrite(dstBuffer, 1, endSize, dstFile) != endSize)
+ EXM_THROW(39, "Write error : cannot write end of frame");
+ compressedfilesize += endSize;
+ } }
/* Release file handlers */
fclose (srcFile);
- if (strcmp(dstFileName,stdoutmark)) fclose (dstFile); /* do not close stdout */
+ if (strcmp(dstFileName,stdoutmark)) fclose (dstFile); /* do not close stdout */
/* Copy owner, file permissions and modification time */
{ stat_t statbuf;
@@ -861,7 +859,11 @@ static unsigned LZ4IO_readLE32 (const void* s)
}
-static unsigned LZ4IO_fwriteSparse(LZ4IO_prefs_t* const prefs, FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips)
+static unsigned
+LZ4IO_fwriteSparse(FILE* file,
+ const void* buffer, size_t bufferSize,
+ int sparseFileSupport,
+ unsigned storedSkips)
{
const size_t sizeT = sizeof(size_t);
const size_t maskT = sizeT -1 ;
@@ -871,7 +873,7 @@ static unsigned LZ4IO_fwriteSparse(LZ4IO_prefs_t* const prefs, FILE* file, const
const size_t* const bufferTEnd = bufferT + bufferSizeT;
const size_t segmentSizeT = (32 KB) / sizeT;
- if (!prefs->sparseFileSupport) { /* normal write */
+ if (!sparseFileSupport) { /* normal write */
size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file);
if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : cannot write decoded block");
return 0;
@@ -919,7 +921,7 @@ static unsigned LZ4IO_fwriteSparse(LZ4IO_prefs_t* const prefs, FILE* file, const
int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR);
if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse");
storedSkips = 0;
- { size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file);
+ { size_t const sizeCheck = fwrite(restPtr, 1, (size_t)(restEnd - restPtr), file);
if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(75, "Write error : cannot write decoded end of block");
} }
}
@@ -929,18 +931,18 @@ static unsigned LZ4IO_fwriteSparse(LZ4IO_prefs_t* const prefs, FILE* file, const
static void LZ4IO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
{
- if (storedSkips>0) { /* implies g_sparseFileSupport>0 */
- int const seekResult = UTIL_fseek(file, storedSkips-1, SEEK_CUR);
- if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n");
- { const char lastZeroByte[1] = { 0 };
- size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file);
- if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n");
- } }
+ if (storedSkips>0) { /* implies sparseFileSupport>0 */
+ const char lastZeroByte[1] = { 0 };
+ if (UTIL_fseek(file, storedSkips-1, SEEK_CUR) != 0)
+ EXM_THROW(69, "Final skip error (sparse file)\n");
+ if (fwrite(lastZeroByte, 1, 1, file) != 1)
+ EXM_THROW(69, "Write error : cannot write last zero\n");
+ }
}
static unsigned g_magicRead = 0; /* out-parameter of LZ4IO_decodeLegacyStream() */
-static unsigned long long LZ4IO_decodeLegacyStream(LZ4IO_prefs_t* const prefs, FILE* finput, FILE* foutput)
+static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput, const LZ4IO_prefs_t* prefs)
{
unsigned long long streamSize = 0;
unsigned storedSkips = 0;
@@ -974,7 +976,7 @@ static unsigned long long LZ4IO_decodeLegacyStream(LZ4IO_prefs_t* const prefs, F
if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !");
streamSize += (unsigned long long)decodeSize;
/* Write Block */
- storedSkips = LZ4IO_fwriteSparse(prefs, foutput, out_buff, (size_t)decodeSize, storedSkips); /* success or die */
+ storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, (size_t)decodeSize, prefs->sparseFileSupport, storedSkips); /* success or die */
} }
if (ferror(finput)) EXM_THROW(54, "Read error : ferror");
@@ -1000,19 +1002,20 @@ typedef struct {
size_t dictBufferSize;
} dRess_t;
-static void LZ4IO_loadDDict(LZ4IO_prefs_t* const prefs, dRess_t* ress) {
+static void LZ4IO_loadDDict(dRess_t* ress, const LZ4IO_prefs_t* const prefs)
+{
if (!prefs->useDictionary) {
ress->dictBuffer = NULL;
ress->dictBufferSize = 0;
return;
}
- ress->dictBuffer = LZ4IO_createDict(prefs, &ress->dictBufferSize);
+ ress->dictBuffer = LZ4IO_createDict(&ress->dictBufferSize, prefs->dictionaryFilename);
if (!ress->dictBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary");
}
static const size_t LZ4IO_dBufferSize = 64 KB;
-static dRess_t LZ4IO_createDResources(LZ4IO_prefs_t* const prefs)
+static dRess_t LZ4IO_createDResources(const LZ4IO_prefs_t* const prefs)
{
dRess_t ress;
@@ -1027,7 +1030,7 @@ static dRess_t LZ4IO_createDResources(LZ4IO_prefs_t* const prefs)
ress.dstBuffer = malloc(ress.dstBufferSize);
if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
- LZ4IO_loadDDict(prefs, &ress);
+ LZ4IO_loadDDict(&ress, prefs);
ress.dstFile = NULL;
return ress;
@@ -1043,7 +1046,10 @@ static void LZ4IO_freeDResources(dRess_t ress)
}
-static unsigned long long LZ4IO_decompressLZ4F(LZ4IO_prefs_t* const prefs, dRess_t ress, FILE* srcFile, FILE* dstFile)
+static unsigned long long
+LZ4IO_decompressLZ4F(dRess_t ress,
+ FILE* const srcFile, FILE* const dstFile,
+ const LZ4IO_prefs_t* const prefs)
{
unsigned long long filesize = 0;
LZ4F_errorCode_t nextToLoad;
@@ -1079,7 +1085,7 @@ static unsigned long long LZ4IO_decompressLZ4F(LZ4IO_prefs_t* const prefs, dRess
/* Write Block */
if (decodedBytes) {
if (!prefs->testMode)
- storedSkips = LZ4IO_fwriteSparse(prefs, dstFile, ress.dstBuffer, decodedBytes, storedSkips);
+ storedSkips = LZ4IO_fwriteSparse(dstFile, ress.dstBuffer, decodedBytes, prefs->sparseFileSupport, storedSkips);
filesize += decodedBytes;
DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20));
}
@@ -1097,22 +1103,30 @@ static unsigned long long LZ4IO_decompressLZ4F(LZ4IO_prefs_t* const prefs, dRess
}
+/* LZ4IO_passThrough:
+ * just output the same content as input, no decoding.
+ * This is a capability of zcat, and by extension lz4cat
+ * MNstore : contain the first MAGICNUMBER_SIZE bytes already read from finput
+ */
#define PTSIZE (64 KB)
#define PTSIZET (PTSIZE / sizeof(size_t))
-static unsigned long long LZ4IO_passThrough(LZ4IO_prefs_t* const prefs, FILE* finput, FILE* foutput, unsigned char MNstore[MAGICNUMBER_SIZE])
+static unsigned long long
+LZ4IO_passThrough(FILE* finput, FILE* foutput,
+ unsigned char MNstore[MAGICNUMBER_SIZE],
+ int sparseFileSupport)
{
size_t buffer[PTSIZET];
size_t readBytes = 1;
unsigned long long total = MAGICNUMBER_SIZE;
unsigned storedSkips = 0;
- size_t const sizeCheck = fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput);
- if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(50, "Pass-through write error");
-
+ if (fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput) != MAGICNUMBER_SIZE) {
+ EXM_THROW(50, "Pass-through write error");
+ }
while (readBytes) {
- readBytes = fread(buffer, 1, PTSIZE, finput);
+ readBytes = fread(buffer, 1, sizeof(buffer), finput);
total += readBytes;
- storedSkips = LZ4IO_fwriteSparse(prefs, foutput, buffer, readBytes, storedSkips);
+ storedSkips = LZ4IO_fwriteSparse(foutput, buffer, readBytes, sparseFileSupport, storedSkips);
}
if (ferror(finput)) EXM_THROW(51, "Read Error");
@@ -1139,7 +1153,10 @@ static int fseek_u32(FILE *fp, unsigned offset, int where)
}
#define ENDOFSTREAM ((unsigned long long)-1)
-static unsigned long long selectDecoder(LZ4IO_prefs_t* const prefs, dRess_t ress, FILE* finput, FILE* foutput)
+static unsigned long long
+selectDecoder(dRess_t ress,
+ FILE* finput, FILE* foutput,
+ const LZ4IO_prefs_t* const prefs)
{
unsigned char MNstore[MAGICNUMBER_SIZE];
unsigned magicNumber;
@@ -1165,10 +1182,10 @@ static unsigned long long selectDecoder(LZ4IO_prefs_t* const prefs, dRess_t ress
switch(magicNumber)
{
case LZ4IO_MAGICNUMBER:
- return LZ4IO_decompressLZ4F(prefs, ress, finput, foutput);
+ return LZ4IO_decompressLZ4F(ress, finput, foutput, prefs);
case LEGACY_MAGICNUMBER:
DISPLAYLEVEL(4, "Detected : Legacy format \n");
- return LZ4IO_decodeLegacyStream(prefs, finput, foutput);
+ return LZ4IO_decodeLegacyStream(finput, foutput, prefs);
case LZ4IO_SKIPPABLE0:
DISPLAYLEVEL(4, "Skipping detected skippable area \n");
{ size_t const nbReadBytes = fread(MNstore, 1, 4, finput);
@@ -1187,7 +1204,7 @@ static unsigned long long selectDecoder(LZ4IO_prefs_t* const prefs, dRess_t ress
/* Wrong magic number at the beginning of 1st stream */
if (!prefs->testMode && prefs->overwrite && prefs->passThrough) {
nbFrames = 0;
- return LZ4IO_passThrough(prefs, finput, foutput, MNstore);
+ return LZ4IO_passThrough(finput, foutput, MNstore, prefs->sparseFileSupport);
}
EXM_THROW(44,"Unrecognized header : file cannot be decoded");
}
@@ -1202,7 +1219,10 @@ static unsigned long long selectDecoder(LZ4IO_prefs_t* const prefs, dRess_t ress
}
-static int LZ4IO_decompressSrcFile(LZ4IO_prefs_t* const prefs, dRess_t ress, const char* input_filename, const char* output_filename)
+static int
+LZ4IO_decompressSrcFile(dRess_t ress,
+ const char* input_filename, const char* output_filename,
+ const LZ4IO_prefs_t* const prefs)
{
FILE* const foutput = ress.dstFile;
unsigned long long filesize = 0;
@@ -1210,11 +1230,12 @@ static int LZ4IO_decompressSrcFile(LZ4IO_prefs_t* const prefs, dRess_t ress, con
/* Init */
FILE* const finput = LZ4IO_openSrcFile(input_filename);
if (finput==NULL) return 1;
+ assert(foutput != NULL);
/* Loop over multiple streams */
for ( ; ; ) { /* endless loop, see break condition */
unsigned long long const decodedSize =
- selectDecoder(prefs, ress, finput, foutput);
+ selectDecoder(ress, finput, foutput, prefs);
if (decodedSize == ENDOFSTREAM) break;
filesize += decodedSize;
}
@@ -1235,7 +1256,10 @@ static int LZ4IO_decompressSrcFile(LZ4IO_prefs_t* const prefs, dRess_t ress, con
}
-static int LZ4IO_decompressDstFile(LZ4IO_prefs_t* const prefs, dRess_t ress, const char* input_filename, const char* output_filename)
+static int
+LZ4IO_decompressDstFile(LZ4IO_prefs_t* const prefs,
+ dRess_t ress,
+ const char* input_filename, const char* output_filename)
{
stat_t statbuf;
int stat_result = 0;
@@ -1247,7 +1271,7 @@ static int LZ4IO_decompressDstFile(LZ4IO_prefs_t* const prefs, dRess_t ress, con
stat_result = 1;
ress.dstFile = foutput;
- LZ4IO_decompressSrcFile(prefs, ress, input_filename, output_filename);
+ LZ4IO_decompressSrcFile(ress, input_filename, output_filename, prefs);
fclose(foutput);
@@ -1298,7 +1322,7 @@ int LZ4IO_decompressMultipleFilenames(LZ4IO_prefs_t* const prefs,
size_t const ifnSize = strlen(inFileNamesTable[i]);
const char* const suffixPtr = inFileNamesTable[i] + ifnSize - suffixSize;
if (!strcmp(suffix, stdoutmark)) {
- missingFiles += LZ4IO_decompressSrcFile(prefs, ress, inFileNamesTable[i], stdoutmark);
+ missingFiles += LZ4IO_decompressSrcFile(ress, inFileNamesTable[i], stdoutmark, prefs);
continue;
}
if (ofnSize <= ifnSize-suffixSize+1) {
@@ -1351,7 +1375,7 @@ typedef struct {
unsigned short allContentSize;
} LZ4IO_cFileInfo_t;
-#define LZ4IO_INIT_CFILEINFO { NULL, 0ULL, 0, LZ4IO_INIT_FRAMEINFO, 1, 1, 1 }
+#define LZ4IO_INIT_CFILEINFO { NULL, 0ULL, 0, LZ4IO_INIT_FRAMEINFO, 1, 1, 1 }
typedef enum { LZ4IO_LZ4F_OK, LZ4IO_format_not_known, LZ4IO_not_a_file } LZ4IO_infoResult;
@@ -1363,9 +1387,11 @@ static const char * LZ4IO_frameTypeNames[] = {"LZ4Frame", "LegacyFrame", "Skippa
returns 0 in case it can't succesfully skip block data.
Assumes SEEK_CUR after frame header.
*/
-static unsigned long long LZ4IO_skipBlocksData(FILE* finput,
- const LZ4F_blockChecksum_t blockChecksumFlag,
- const LZ4F_contentChecksum_t contentChecksumFlag) {
+static unsigned long long
+LZ4IO_skipBlocksData(FILE* finput,
+ const LZ4F_blockChecksum_t blockChecksumFlag,
+ const LZ4F_contentChecksum_t contentChecksumFlag)
+{
unsigned char blockInfo[LZ4F_BLOCK_HEADER_SIZE];
unsigned long long totalBlocksSize = 0;
for (;;) {
@@ -1374,8 +1400,7 @@ static unsigned long long LZ4IO_skipBlocksData(FILE* finput,
return 0;
}
totalBlocksSize += LZ4F_BLOCK_HEADER_SIZE;
- {
- const unsigned long nextCBlockSize = LZ4IO_readLE32(&blockInfo) & 0x7FFFFFFFU;
+ { const unsigned long nextCBlockSize = LZ4IO_readLE32(&blockInfo) & 0x7FFFFFFFU;
const unsigned long nextBlock = nextCBlockSize + (blockChecksumFlag * LZ4F_BLOCK_CHECKSUM_SIZE);
if (nextCBlockSize == 0) {
/* Reached EndMark */
@@ -1390,11 +1415,9 @@ static unsigned long long LZ4IO_skipBlocksData(FILE* finput,
}
totalBlocksSize += nextBlock;
/* skip to the next block */
- if (UTIL_fseek(finput, nextBlock, SEEK_CUR) != 0) {
- return 0;
- }
- }
- }
+ assert(nextBlock < LONG_MAX);
+ if (UTIL_fseek(finput, (long)nextBlock, SEEK_CUR) != 0) return 0;
+ } }
return totalBlocksSize;
}
@@ -1405,7 +1428,8 @@ static unsigned long long LZ4IO_skipBlocksData(FILE* finput,
This works as long as legacy block header size = magic number size.
Assumes SEEK_CUR after frame header.
*/
-static unsigned long long LZ4IO_skipLegacyBlocksData(FILE* finput) {
+static unsigned long long LZ4IO_skipLegacyBlocksData(FILE* finput)
+{
unsigned char blockInfo[LZIO_LEGACY_BLOCK_HEADER_SIZE];
unsigned long long totalBlocksSize = 0;
LZ4IO_STATIC_ASSERT(LZIO_LEGACY_BLOCK_HEADER_SIZE == MAGICNUMBER_SIZE);
@@ -1428,14 +1452,15 @@ static unsigned long long LZ4IO_skipLegacyBlocksData(FILE* finput) {
/* skip to the next block */
if (UTIL_fseek(finput, nextCBlockSize, SEEK_CUR) != 0) {
return 0;
- }
- }
- }
+ } } }
return totalBlocksSize;
}
-/* buffer : must be a valid memory area of at least 4 bytes */
-const char* LZ4IO_blockTypeID(int sizeID, int blockMode, char* buffer) {
+/* LZ4IO_blockTypeID:
+ * return human-readable block type, following command line convention
+ * buffer : must be a valid memory area of at least 4 bytes */
+const char* LZ4IO_blockTypeID(LZ4F_blockSizeID_t sizeID, LZ4F_blockMode_t blockMode, char buffer[4])
+{
buffer[0] = 'B';
assert(sizeID >= 4); assert(sizeID <= 7);
buffer[1] = (char)(sizeID + '0');
@@ -1445,7 +1470,8 @@ const char* LZ4IO_blockTypeID(int sizeID, int blockMode, char* buffer) {
}
/* buffer : must be valid memory area of at least 10 bytes */
-static const char* LZ4IO_toHuman(long double size, char *buf) {
+static const char* LZ4IO_toHuman(long double size, char *buf)
+{
const char units[] = {"\0KMGTPEZY"};
size_t i = 0;
for (; size >= 1024; i++) size /= 1024;
@@ -1454,14 +1480,15 @@ static const char* LZ4IO_toHuman(long double size, char *buf) {
}
/* Get filename without path prefix */
-static const char* LZ4IO_baseName(const char* input_filename) {
+static const char* LZ4IO_baseName(const char* input_filename)
+{
const char* b = strrchr(input_filename, '/');
if (!b) b = strrchr(input_filename, '\\');
if (!b) return input_filename;
return b + 1;
}
-/* Report frame/s information in verbose mode.
+/* Report frame/s information (--list) in verbose mode (-v).
* Will populate file info with fileName and frameSummary where applicable.
* - TODO :
* + report nb of blocks, hence max. possible decompressed size (when not reported in header)
@@ -1480,11 +1507,12 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam
LZ4IO_frameInfo_t frameInfo = LZ4IO_INIT_FRAMEINFO;
unsigned magicNumber;
/* Get MagicNumber */
- size_t nbReadBytes = fread(buffer, 1, MAGICNUMBER_SIZE, finput);
- if (nbReadBytes == 0) { break; } /* EOF */
- result = LZ4IO_format_not_known; /* default result (error) */
- if (nbReadBytes != MAGICNUMBER_SIZE)
- EXM_THROW(40, "Unrecognized header : Magic Number unreadable");
+ { size_t const nbReadBytes = fread(buffer, 1, MAGICNUMBER_SIZE, finput);
+ if (nbReadBytes == 0) { break; } /* EOF */
+ result = LZ4IO_format_not_known; /* default result (error) */
+ if (nbReadBytes != MAGICNUMBER_SIZE) {
+ EXM_THROW(40, "Unrecognized header : Magic Number unreadable");
+ } }
magicNumber = LZ4IO_readLE32(buffer); /* Little Endian format */
if (LZ4IO_isSkippableMagicNumber(magicNumber))
magicNumber = LZ4IO_SKIPPABLE0; /* fold skippable magic numbers */
@@ -1497,56 +1525,49 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam
if (!readBytes || ferror(finput)) EXM_THROW(71, "Error reading %s", input_filename);
}
{ size_t hSize = LZ4F_headerSize(&buffer, LZ4F_HEADER_SIZE_MIN);
- if (!LZ4F_isError(hSize)) {
- if (hSize > (LZ4F_HEADER_SIZE_MIN + MAGICNUMBER_SIZE)) {
- /* We've already read LZ4F_HEADER_SIZE_MIN so read any extra until hSize*/
- const size_t readBytes = fread(buffer + LZ4F_HEADER_SIZE_MIN, 1, hSize - LZ4F_HEADER_SIZE_MIN, finput);
- if (!readBytes || ferror(finput)) EXM_THROW(72, "Error reading %s", input_filename);
- }
- /* Create decompression context */
- { LZ4F_dctx* dctx;
- unsigned isError = LZ4F_isError(LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION));
- if (!isError) {
- isError = LZ4F_isError(LZ4F_getFrameInfo(dctx, &frameInfo.lz4FrameInfo, buffer, &hSize));
- LZ4F_freeDecompressionContext(dctx);
- if (!isError) {
- if ((cfinfo->frameSummary.lz4FrameInfo.blockSizeID != frameInfo.lz4FrameInfo.blockSizeID ||
- cfinfo->frameSummary.lz4FrameInfo.blockMode != frameInfo.lz4FrameInfo.blockMode)
- && cfinfo->frameCount != 0)
- cfinfo->eqBlockTypes = 0;
- { const unsigned long long totalBlocksSize = LZ4IO_skipBlocksData(finput,
- frameInfo.lz4FrameInfo.blockChecksumFlag,
- frameInfo.lz4FrameInfo.contentChecksumFlag);
- if (totalBlocksSize) {
- char bTypeBuffer[5];
- LZ4IO_blockTypeID(frameInfo.lz4FrameInfo.blockSizeID, frameInfo.lz4FrameInfo.blockMode, bTypeBuffer);
- DISPLAYLEVEL(3, " %6llu %14s %5s %8s",
- cfinfo->frameCount + 1,
- LZ4IO_frameTypeNames[frameInfo.frameType],
- bTypeBuffer,
- frameInfo.lz4FrameInfo.contentChecksumFlag ? "XXH32" : "-");
- if (frameInfo.lz4FrameInfo.contentSize) {
- { double const ratio = (double)(totalBlocksSize + hSize) / frameInfo.lz4FrameInfo.contentSize * 100;
- DISPLAYLEVEL(3, " %20llu %20llu %9.2f%%\n",
- totalBlocksSize + hSize,
- frameInfo.lz4FrameInfo.contentSize,
- ratio);
- }
- /* Now we've consumed frameInfo we can use it to store the total contentSize */
- frameInfo.lz4FrameInfo.contentSize += cfinfo->frameSummary.lz4FrameInfo.contentSize;
- }
- else {
- DISPLAYLEVEL(3, " %20llu %20s %9s \n", totalBlocksSize + hSize, "-", "-");
- cfinfo->allContentSize = 0;
- }
- result = LZ4IO_LZ4F_OK;
+ if (LZ4F_isError(hSize)) break;
+ if (hSize > (LZ4F_HEADER_SIZE_MIN + MAGICNUMBER_SIZE)) {
+ /* We've already read LZ4F_HEADER_SIZE_MIN so read any extra until hSize*/
+ const size_t readBytes = fread(buffer + LZ4F_HEADER_SIZE_MIN, 1, hSize - LZ4F_HEADER_SIZE_MIN, finput);
+ if (!readBytes || ferror(finput)) EXM_THROW(72, "Error reading %s", input_filename);
+ }
+ /* Create decompression context */
+ { LZ4F_dctx* dctx;
+ if ( LZ4F_isError(LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION)) ) break;
+ { unsigned const frameInfoError = LZ4F_isError(LZ4F_getFrameInfo(dctx, &frameInfo.lz4FrameInfo, buffer, &hSize));
+ LZ4F_freeDecompressionContext(dctx);
+ if (frameInfoError) break;
+ if ((cfinfo->frameSummary.lz4FrameInfo.blockSizeID != frameInfo.lz4FrameInfo.blockSizeID ||
+ cfinfo->frameSummary.lz4FrameInfo.blockMode != frameInfo.lz4FrameInfo.blockMode)
+ && cfinfo->frameCount != 0)
+ cfinfo->eqBlockTypes = 0;
+ { const unsigned long long totalBlocksSize = LZ4IO_skipBlocksData(finput,
+ frameInfo.lz4FrameInfo.blockChecksumFlag,
+ frameInfo.lz4FrameInfo.contentChecksumFlag);
+ if (totalBlocksSize) {
+ char bTypeBuffer[5];
+ LZ4IO_blockTypeID(frameInfo.lz4FrameInfo.blockSizeID, frameInfo.lz4FrameInfo.blockMode, bTypeBuffer);
+ DISPLAYLEVEL(3, " %6llu %14s %5s %8s",
+ cfinfo->frameCount + 1,
+ LZ4IO_frameTypeNames[frameInfo.frameType],
+ bTypeBuffer,
+ frameInfo.lz4FrameInfo.contentChecksumFlag ? "XXH32" : "-");
+ if (frameInfo.lz4FrameInfo.contentSize) {
+ { double const ratio = (double)(totalBlocksSize + hSize) / frameInfo.lz4FrameInfo.contentSize * 100;
+ DISPLAYLEVEL(3, " %20llu %20llu %9.2f%%\n",
+ totalBlocksSize + hSize,
+ frameInfo.lz4FrameInfo.contentSize,
+ ratio);
}
+ /* Now we've consumed frameInfo we can use it to store the total contentSize */
+ frameInfo.lz4FrameInfo.contentSize += cfinfo->frameSummary.lz4FrameInfo.contentSize;
}
- }
- }
- }
- }
- }
+ else {
+ DISPLAYLEVEL(3, " %20llu %20s %9s \n", totalBlocksSize + hSize, "-", "-");
+ cfinfo->allContentSize = 0;
+ }
+ result = LZ4IO_LZ4F_OK;
+ } } } } }
break;
case LEGACY_MAGICNUMBER:
frameInfo.frameType = legacyFrame;
@@ -1569,7 +1590,7 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam
if (cfinfo->frameSummary.frameType != skippableFrame && cfinfo->frameCount != 0) cfinfo->eqFrameTypes = 0;
cfinfo->eqBlockTypes = 0;
cfinfo->allContentSize = 0;
- { nbReadBytes = fread(buffer, 1, 4, finput);
+ { size_t const nbReadBytes = fread(buffer, 1, 4, finput);
if (nbReadBytes != 4)
EXM_THROW(42, "Stream error : skippable size unreadable");
}
@@ -1594,12 +1615,10 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam
}
break;
}
- if (result != LZ4IO_LZ4F_OK) {
- break;
- }
+ if (result != LZ4IO_LZ4F_OK) break;
cfinfo->frameSummary = frameInfo;
cfinfo->frameCount++;
- }
+ } /* while (!feof(finput)) */
fclose(finput);
return result;
}
diff --git a/tests/Makefile b/tests/Makefile
index 476849e..6eee132 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -420,7 +420,7 @@ test-lz4-hugefile: lz4 datagen
test-lz4-testmode: lz4 datagen
@echo "\n ---- bench mode ----"
- $(LZ4) -bi1
+ $(LZ4) -bi0
@echo "\n ---- test mode ----"
! $(DATAGEN) | $(LZ4) -t
! $(DATAGEN) | $(LZ4) -tf