diff options
Diffstat (limited to 'programs/lz4io.c')
| -rw-r--r-- | programs/lz4io.c | 383 |
1 files changed, 201 insertions, 182 deletions
diff --git a/programs/lz4io.c b/programs/lz4io.c index 3ffc519..975f590 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -111,20 +111,20 @@ static clock_t g_time = 0; **************************************/ struct LZ4IO_prefs_s { - int passThrough; - int overwrite; - int testMode; - int blockSizeId; - size_t blockSize; - int blockChecksum; - int streamChecksum; - int blockIndependence; - int sparseFileSupport; - int contentSizeFlag; - int useDictionary; - unsigned favorDecSpeed; - const char* dictionaryFilename; - int removeSrcFile; + int passThrough; + int overwrite; + int testMode; + int blockSizeId; + size_t blockSize; + int blockChecksum; + int streamChecksum; + int blockIndependence; + int sparseFileSupport; + int contentSizeFlag; + int useDictionary; + unsigned favorDecSpeed; + const char* dictionaryFilename; + int removeSrcFile; }; /************************************** @@ -325,6 +325,7 @@ static FILE* LZ4IO_openSrcFile(const char* srcFileName) } /** FIO_openDstFile() : + * prefs is writable, because sparseFileSupport might be updated. * condition : `dstFileName` must be non-NULL. * @result : FILE* to `dstFileName`, or NULL if it fails */ static FILE* LZ4IO_openDstFile(LZ4IO_prefs_t* const prefs, const char* dstFileName) @@ -333,18 +334,19 @@ static FILE* LZ4IO_openDstFile(LZ4IO_prefs_t* const prefs, const char* dstFileNa assert(dstFileName != NULL); if (!strcmp (dstFileName, stdoutmark)) { - DISPLAYLEVEL(4,"Using stdout for output\n"); + DISPLAYLEVEL(4, "Using stdout for output \n"); f = stdout; SET_BINARY_MODE(stdout); if (prefs->sparseFileSupport==1) { prefs->sparseFileSupport = 0; - DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); + DISPLAYLEVEL(4, "Sparse File Support automatically disabled on stdout ;" + " to force-enable it, add --sparse command \n"); } } else { if (!prefs->overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */ - f = fopen( dstFileName, "rb" ); - if (f != NULL) { /* dest exists, prompt for overwrite authorization */ - fclose(f); + FILE* const testf = fopen( dstFileName, "rb" ); + if (testf != NULL) { /* dest exists, prompt for overwrite authorization */ + fclose(testf); if (g_displayLevel <= 1) { /* No interaction possible */ DISPLAY("%s already exists; not overwritten \n", dstFileName); return NULL; @@ -392,7 +394,9 @@ static int LZ4IO_LZ4_compress(const char* src, char* dst, int srcSize, int dstSi /* LZ4IO_compressFilename_Legacy : * This function is intentionally "hidden" (not published in .h) * It generates compressed streams using the old 'legacy' format */ -int LZ4IO_compressFilename_Legacy(LZ4IO_prefs_t* const prefs, const char* input_filename, const char* output_filename, int compressionlevel) +int LZ4IO_compressFilename_Legacy(LZ4IO_prefs_t* const prefs, + const char* input_filename, const char* output_filename, + int compressionlevel) { typedef int (*compress_f)(const char* src, char* dst, int srcSize, int dstSize, int cLevel); compress_f const compressionFunction = (compressionlevel < 3) ? LZ4IO_LZ4_compress : LZ4_compress_HC; @@ -424,23 +428,22 @@ int LZ4IO_compressFilename_Legacy(LZ4IO_prefs_t* const prefs, const char* input_ /* Write Archive Header */ LZ4IO_writeLE32(out_buff, LEGACY_MAGICNUMBER); - { size_t const writeSize = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput); - if (writeSize != MAGICNUMBER_SIZE) - EXM_THROW(22, "Write error : cannot write header"); - } + if (fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput) != MAGICNUMBER_SIZE) + EXM_THROW(22, "Write error : cannot write header"); /* Main Loop */ while (1) { int outSize; /* Read Block */ size_t const inSize = fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput); - assert(inSize <= LEGACY_BLOCKSIZE); if (inSize == 0) break; + assert(inSize <= LEGACY_BLOCKSIZE); filesize += inSize; /* Compress Block */ outSize = compressionFunction(in_buff, out_buff+4, (int)inSize, outBuffSize, compressionlevel); - compressedfilesize += outSize+4; + assert(outSize >= 0); + compressedfilesize += (unsigned long long)outSize+4; DISPLAYUPDATE(2, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); @@ -448,9 +451,8 @@ int LZ4IO_compressFilename_Legacy(LZ4IO_prefs_t* const prefs, const char* input_ assert(outSize > 0); assert(outSize < outBuffSize); LZ4IO_writeLE32(out_buff, (unsigned)outSize); - { size_t const writeSize = fwrite(out_buff, 1, outSize+4, foutput); - if (writeSize != (size_t)(outSize+4)) - EXM_THROW(24, "Write error : cannot write compressed block"); + if (fwrite(out_buff, 1, (size_t)outSize+4, foutput) != (size_t)(outSize+4)) { + EXM_THROW(24, "Write error : cannot write compressed block"); } } if (ferror(finput)) EXM_THROW(25, "Error while reading %s ", input_filename); @@ -537,22 +539,20 @@ typedef struct { LZ4F_CDict* cdict; } cRess_t; -static void* LZ4IO_createDict(LZ4IO_prefs_t* const prefs, size_t *dictSize) { +static void* LZ4IO_createDict(size_t* dictSize, const char* const dictFilename) +{ size_t readSize; size_t dictEnd = 0; size_t dictLen = 0; size_t dictStart; size_t circularBufSize = LZ4_MAX_DICT_SIZE; - char* circularBuf; - char* dictBuf; - const char* dictFilename = prefs->dictionaryFilename; + char* circularBuf = (char*)malloc(circularBufSize); + char* dictBuf; FILE* dictFile; + if (!circularBuf) EXM_THROW(25, "Allocation error : not enough memory for circular buffer"); if (!dictFilename) EXM_THROW(25, "Dictionary error : no filename provided"); - circularBuf = (char *) malloc(circularBufSize); - if (!circularBuf) EXM_THROW(25, "Allocation error : not enough memory"); - dictFile = LZ4IO_openSrcFile(dictFilename); if (!dictFile) EXM_THROW(25, "Dictionary error : could not open dictionary file"); @@ -582,7 +582,7 @@ static void* LZ4IO_createDict(LZ4IO_prefs_t* const prefs, size_t *dictSize) { circularBuf = NULL; } else { /* Otherwise, we will alloc a new buffer and copy our dict into that. */ - dictBuf = (char *) malloc(dictLen ? dictLen : 1); + dictBuf = (char *)malloc(dictLen ? dictLen : 1); if (!dictBuf) EXM_THROW(25, "Allocation error : not enough memory"); memcpy(dictBuf, circularBuf + dictStart, circularBufSize - dictStart); @@ -595,14 +595,13 @@ static void* LZ4IO_createDict(LZ4IO_prefs_t* const prefs, size_t *dictSize) { return dictBuf; } -static LZ4F_CDict* LZ4IO_createCDict(LZ4IO_prefs_t* const prefs) { +static LZ4F_CDict* LZ4IO_createCDict(const LZ4IO_prefs_t* const prefs) +{ size_t dictionarySize; void* dictionaryBuffer; LZ4F_CDict* cdict; - if (!prefs->useDictionary) { - return NULL; - } - dictionaryBuffer = LZ4IO_createDict(prefs, &dictionarySize); + if (!prefs->useDictionary) return NULL; + dictionaryBuffer = LZ4IO_createDict(&dictionarySize, prefs->dictionaryFilename); if (!dictionaryBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary"); cdict = LZ4F_createCDict(dictionaryBuffer, dictionarySize); free(dictionaryBuffer); @@ -643,6 +642,7 @@ static void LZ4IO_freeCResources(cRess_t ress) /* * LZ4IO_compressFilename_extRess() + * io_prefs is mutable, as it may update sparseFileSupport * result : 0 : compression completed correctly * 1 : missing or pb opening srcFileName */ @@ -653,7 +653,6 @@ LZ4IO_compressFilename_extRess(LZ4IO_prefs_t* const io_prefs, cRess_t ress, { unsigned long long filesize = 0; unsigned long long compressedfilesize = 0; - FILE* srcFile; FILE* dstFile; void* const srcBuffer = ress.srcBuffer; void* const dstBuffer = ress.dstBuffer; @@ -664,13 +663,12 @@ LZ4IO_compressFilename_extRess(LZ4IO_prefs_t* const io_prefs, cRess_t ress, LZ4F_preferences_t prefs; /* Init */ - srcFile = LZ4IO_openSrcFile(srcFileName); + FILE* const srcFile = LZ4IO_openSrcFile(srcFileName); if (srcFile == NULL) return 1; dstFile = LZ4IO_openDstFile(io_prefs, dstFileName); if (dstFile == NULL) { fclose(srcFile); return 1; } memset(&prefs, 0, sizeof(prefs)); - /* Set compression parameters */ prefs.autoFlush = 1; prefs.compressionLevel = compressionLevel; @@ -694,41 +692,41 @@ LZ4IO_compressFilename_extRess(LZ4IO_prefs_t* const io_prefs, cRess_t ress, /* single-block file */ if (readSize < blockSize) { /* Compress in single pass */ - size_t cSize = LZ4F_compressFrame_usingCDict(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs); - if (LZ4F_isError(cSize)) EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize)); + size_t const cSize = LZ4F_compressFrame_usingCDict(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs); + if (LZ4F_isError(cSize)) + EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize)); compressedfilesize = cSize; DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (unsigned)(filesize>>20), (double)compressedfilesize/(filesize+!filesize)*100); /* avoid division by zero */ /* Write Block */ - { size_t const sizeCheck = fwrite(dstBuffer, 1, cSize, dstFile); - if (sizeCheck!=cSize) EXM_THROW(32, "Write error : cannot write compressed block"); + if (fwrite(dstBuffer, 1, cSize, dstFile) != cSize) { + EXM_THROW(32, "Write error : failed writing single-block compressed frame"); } } else /* multiple-blocks file */ { - /* Write Archive Header */ - size_t headerSize = LZ4F_compressBegin_usingCDict(ctx, dstBuffer, dstBufferSize, ress.cdict, &prefs); + /* Write Frame Header */ + size_t const headerSize = LZ4F_compressBegin_usingCDict(ctx, dstBuffer, dstBufferSize, ress.cdict, &prefs); if (LZ4F_isError(headerSize)) EXM_THROW(33, "File header generation failed : %s", LZ4F_getErrorName(headerSize)); - { size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); - if (sizeCheck!=headerSize) EXM_THROW(34, "Write error : cannot write header"); } + if (fwrite(dstBuffer, 1, headerSize, dstFile) != headerSize) + EXM_THROW(34, "Write error : cannot write header"); compressedfilesize += headerSize; - /* Main Loop */ + /* Main Loop - one block at a time */ while (readSize>0) { - size_t outSize; - - /* Compress Block */ - outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL); - if (LZ4F_isError(outSize)) EXM_THROW(35, "Compression failed : %s", LZ4F_getErrorName(outSize)); + size_t const outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL); + if (LZ4F_isError(outSize)) + EXM_THROW(35, "Compression failed : %s", LZ4F_getErrorName(outSize)); compressedfilesize += outSize; - DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100); + DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", + (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100); /* Write Block */ - { size_t const sizeCheck = fwrite(dstBuffer, 1, outSize, dstFile); - if (sizeCheck!=outSize) EXM_THROW(36, "Write error : cannot write compressed block"); } + if (fwrite(dstBuffer, 1, outSize, dstFile) != outSize) + EXM_THROW(36, "Write error : cannot write compressed block"); /* Read next block */ readSize = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile); @@ -736,18 +734,18 @@ LZ4IO_compressFilename_extRess(LZ4IO_prefs_t* const io_prefs, cRess_t ress, } if (ferror(srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName); - /* End of Stream mark */ - headerSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL); - if (LZ4F_isError(headerSize)) EXM_THROW(38, "End of file generation failed : %s", LZ4F_getErrorName(headerSize)); - - { size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); - if (sizeCheck!=headerSize) EXM_THROW(39, "Write error : cannot write end of stream"); } - compressedfilesize += headerSize; - } + /* End of Frame mark */ + { size_t const endSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL); + if (LZ4F_isError(endSize)) + EXM_THROW(38, "End of frame error : %s", LZ4F_getErrorName(endSize)); + if (fwrite(dstBuffer, 1, endSize, dstFile) != endSize) + EXM_THROW(39, "Write error : cannot write end of frame"); + compressedfilesize += endSize; + } } /* Release file handlers */ fclose (srcFile); - if (strcmp(dstFileName,stdoutmark)) fclose (dstFile); /* do not close stdout */ + if (strcmp(dstFileName,stdoutmark)) fclose (dstFile); /* do not close stdout */ /* Copy owner, file permissions and modification time */ { stat_t statbuf; @@ -861,7 +859,11 @@ static unsigned LZ4IO_readLE32 (const void* s) } -static unsigned LZ4IO_fwriteSparse(LZ4IO_prefs_t* const prefs, FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips) +static unsigned +LZ4IO_fwriteSparse(FILE* file, + const void* buffer, size_t bufferSize, + int sparseFileSupport, + unsigned storedSkips) { const size_t sizeT = sizeof(size_t); const size_t maskT = sizeT -1 ; @@ -871,7 +873,7 @@ static unsigned LZ4IO_fwriteSparse(LZ4IO_prefs_t* const prefs, FILE* file, const const size_t* const bufferTEnd = bufferT + bufferSizeT; const size_t segmentSizeT = (32 KB) / sizeT; - if (!prefs->sparseFileSupport) { /* normal write */ + if (!sparseFileSupport) { /* normal write */ size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : cannot write decoded block"); return 0; @@ -919,7 +921,7 @@ static unsigned LZ4IO_fwriteSparse(LZ4IO_prefs_t* const prefs, FILE* file, const int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR); if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse"); storedSkips = 0; - { size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file); + { size_t const sizeCheck = fwrite(restPtr, 1, (size_t)(restEnd - restPtr), file); if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(75, "Write error : cannot write decoded end of block"); } } } @@ -929,18 +931,18 @@ static unsigned LZ4IO_fwriteSparse(LZ4IO_prefs_t* const prefs, FILE* file, const static void LZ4IO_fwriteSparseEnd(FILE* file, unsigned storedSkips) { - if (storedSkips>0) { /* implies g_sparseFileSupport>0 */ - int const seekResult = UTIL_fseek(file, storedSkips-1, SEEK_CUR); - if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n"); - { const char lastZeroByte[1] = { 0 }; - size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file); - if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n"); - } } + if (storedSkips>0) { /* implies sparseFileSupport>0 */ + const char lastZeroByte[1] = { 0 }; + if (UTIL_fseek(file, storedSkips-1, SEEK_CUR) != 0) + EXM_THROW(69, "Final skip error (sparse file)\n"); + if (fwrite(lastZeroByte, 1, 1, file) != 1) + EXM_THROW(69, "Write error : cannot write last zero\n"); + } } static unsigned g_magicRead = 0; /* out-parameter of LZ4IO_decodeLegacyStream() */ -static unsigned long long LZ4IO_decodeLegacyStream(LZ4IO_prefs_t* const prefs, FILE* finput, FILE* foutput) +static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput, const LZ4IO_prefs_t* prefs) { unsigned long long streamSize = 0; unsigned storedSkips = 0; @@ -974,7 +976,7 @@ static unsigned long long LZ4IO_decodeLegacyStream(LZ4IO_prefs_t* const prefs, F if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !"); streamSize += (unsigned long long)decodeSize; /* Write Block */ - storedSkips = LZ4IO_fwriteSparse(prefs, foutput, out_buff, (size_t)decodeSize, storedSkips); /* success or die */ + storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, (size_t)decodeSize, prefs->sparseFileSupport, storedSkips); /* success or die */ } } if (ferror(finput)) EXM_THROW(54, "Read error : ferror"); @@ -1000,19 +1002,20 @@ typedef struct { size_t dictBufferSize; } dRess_t; -static void LZ4IO_loadDDict(LZ4IO_prefs_t* const prefs, dRess_t* ress) { +static void LZ4IO_loadDDict(dRess_t* ress, const LZ4IO_prefs_t* const prefs) +{ if (!prefs->useDictionary) { ress->dictBuffer = NULL; ress->dictBufferSize = 0; return; } - ress->dictBuffer = LZ4IO_createDict(prefs, &ress->dictBufferSize); + ress->dictBuffer = LZ4IO_createDict(&ress->dictBufferSize, prefs->dictionaryFilename); if (!ress->dictBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary"); } static const size_t LZ4IO_dBufferSize = 64 KB; -static dRess_t LZ4IO_createDResources(LZ4IO_prefs_t* const prefs) +static dRess_t LZ4IO_createDResources(const LZ4IO_prefs_t* const prefs) { dRess_t ress; @@ -1027,7 +1030,7 @@ static dRess_t LZ4IO_createDResources(LZ4IO_prefs_t* const prefs) ress.dstBuffer = malloc(ress.dstBufferSize); if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory"); - LZ4IO_loadDDict(prefs, &ress); + LZ4IO_loadDDict(&ress, prefs); ress.dstFile = NULL; return ress; @@ -1043,7 +1046,10 @@ static void LZ4IO_freeDResources(dRess_t ress) } -static unsigned long long LZ4IO_decompressLZ4F(LZ4IO_prefs_t* const prefs, dRess_t ress, FILE* srcFile, FILE* dstFile) +static unsigned long long +LZ4IO_decompressLZ4F(dRess_t ress, + FILE* const srcFile, FILE* const dstFile, + const LZ4IO_prefs_t* const prefs) { unsigned long long filesize = 0; LZ4F_errorCode_t nextToLoad; @@ -1079,7 +1085,7 @@ static unsigned long long LZ4IO_decompressLZ4F(LZ4IO_prefs_t* const prefs, dRess /* Write Block */ if (decodedBytes) { if (!prefs->testMode) - storedSkips = LZ4IO_fwriteSparse(prefs, dstFile, ress.dstBuffer, decodedBytes, storedSkips); + storedSkips = LZ4IO_fwriteSparse(dstFile, ress.dstBuffer, decodedBytes, prefs->sparseFileSupport, storedSkips); filesize += decodedBytes; DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20)); } @@ -1097,22 +1103,30 @@ static unsigned long long LZ4IO_decompressLZ4F(LZ4IO_prefs_t* const prefs, dRess } +/* LZ4IO_passThrough: + * just output the same content as input, no decoding. + * This is a capability of zcat, and by extension lz4cat + * MNstore : contain the first MAGICNUMBER_SIZE bytes already read from finput + */ #define PTSIZE (64 KB) #define PTSIZET (PTSIZE / sizeof(size_t)) -static unsigned long long LZ4IO_passThrough(LZ4IO_prefs_t* const prefs, FILE* finput, FILE* foutput, unsigned char MNstore[MAGICNUMBER_SIZE]) +static unsigned long long +LZ4IO_passThrough(FILE* finput, FILE* foutput, + unsigned char MNstore[MAGICNUMBER_SIZE], + int sparseFileSupport) { size_t buffer[PTSIZET]; size_t readBytes = 1; unsigned long long total = MAGICNUMBER_SIZE; unsigned storedSkips = 0; - size_t const sizeCheck = fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput); - if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(50, "Pass-through write error"); - + if (fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput) != MAGICNUMBER_SIZE) { + EXM_THROW(50, "Pass-through write error"); + } while (readBytes) { - readBytes = fread(buffer, 1, PTSIZE, finput); + readBytes = fread(buffer, 1, sizeof(buffer), finput); total += readBytes; - storedSkips = LZ4IO_fwriteSparse(prefs, foutput, buffer, readBytes, storedSkips); + storedSkips = LZ4IO_fwriteSparse(foutput, buffer, readBytes, sparseFileSupport, storedSkips); } if (ferror(finput)) EXM_THROW(51, "Read Error"); @@ -1139,7 +1153,10 @@ static int fseek_u32(FILE *fp, unsigned offset, int where) } #define ENDOFSTREAM ((unsigned long long)-1) -static unsigned long long selectDecoder(LZ4IO_prefs_t* const prefs, dRess_t ress, FILE* finput, FILE* foutput) +static unsigned long long +selectDecoder(dRess_t ress, + FILE* finput, FILE* foutput, + const LZ4IO_prefs_t* const prefs) { unsigned char MNstore[MAGICNUMBER_SIZE]; unsigned magicNumber; @@ -1165,10 +1182,10 @@ static unsigned long long selectDecoder(LZ4IO_prefs_t* const prefs, dRess_t ress switch(magicNumber) { case LZ4IO_MAGICNUMBER: - return LZ4IO_decompressLZ4F(prefs, ress, finput, foutput); + return LZ4IO_decompressLZ4F(ress, finput, foutput, prefs); case LEGACY_MAGICNUMBER: DISPLAYLEVEL(4, "Detected : Legacy format \n"); - return LZ4IO_decodeLegacyStream(prefs, finput, foutput); + return LZ4IO_decodeLegacyStream(finput, foutput, prefs); case LZ4IO_SKIPPABLE0: DISPLAYLEVEL(4, "Skipping detected skippable area \n"); { size_t const nbReadBytes = fread(MNstore, 1, 4, finput); @@ -1187,7 +1204,7 @@ static unsigned long long selectDecoder(LZ4IO_prefs_t* const prefs, dRess_t ress /* Wrong magic number at the beginning of 1st stream */ if (!prefs->testMode && prefs->overwrite && prefs->passThrough) { nbFrames = 0; - return LZ4IO_passThrough(prefs, finput, foutput, MNstore); + return LZ4IO_passThrough(finput, foutput, MNstore, prefs->sparseFileSupport); } EXM_THROW(44,"Unrecognized header : file cannot be decoded"); } @@ -1202,7 +1219,10 @@ static unsigned long long selectDecoder(LZ4IO_prefs_t* const prefs, dRess_t ress } -static int LZ4IO_decompressSrcFile(LZ4IO_prefs_t* const prefs, dRess_t ress, const char* input_filename, const char* output_filename) +static int +LZ4IO_decompressSrcFile(dRess_t ress, + const char* input_filename, const char* output_filename, + const LZ4IO_prefs_t* const prefs) { FILE* const foutput = ress.dstFile; unsigned long long filesize = 0; @@ -1210,11 +1230,12 @@ static int LZ4IO_decompressSrcFile(LZ4IO_prefs_t* const prefs, dRess_t ress, con /* Init */ FILE* const finput = LZ4IO_openSrcFile(input_filename); if (finput==NULL) return 1; + assert(foutput != NULL); /* Loop over multiple streams */ for ( ; ; ) { /* endless loop, see break condition */ unsigned long long const decodedSize = - selectDecoder(prefs, ress, finput, foutput); + selectDecoder(ress, finput, foutput, prefs); if (decodedSize == ENDOFSTREAM) break; filesize += decodedSize; } @@ -1235,7 +1256,10 @@ static int LZ4IO_decompressSrcFile(LZ4IO_prefs_t* const prefs, dRess_t ress, con } -static int LZ4IO_decompressDstFile(LZ4IO_prefs_t* const prefs, dRess_t ress, const char* input_filename, const char* output_filename) +static int +LZ4IO_decompressDstFile(LZ4IO_prefs_t* const prefs, + dRess_t ress, + const char* input_filename, const char* output_filename) { stat_t statbuf; int stat_result = 0; @@ -1247,7 +1271,7 @@ static int LZ4IO_decompressDstFile(LZ4IO_prefs_t* const prefs, dRess_t ress, con stat_result = 1; ress.dstFile = foutput; - LZ4IO_decompressSrcFile(prefs, ress, input_filename, output_filename); + LZ4IO_decompressSrcFile(ress, input_filename, output_filename, prefs); fclose(foutput); @@ -1298,7 +1322,7 @@ int LZ4IO_decompressMultipleFilenames(LZ4IO_prefs_t* const prefs, size_t const ifnSize = strlen(inFileNamesTable[i]); const char* const suffixPtr = inFileNamesTable[i] + ifnSize - suffixSize; if (!strcmp(suffix, stdoutmark)) { - missingFiles += LZ4IO_decompressSrcFile(prefs, ress, inFileNamesTable[i], stdoutmark); + missingFiles += LZ4IO_decompressSrcFile(ress, inFileNamesTable[i], stdoutmark, prefs); continue; } if (ofnSize <= ifnSize-suffixSize+1) { @@ -1351,7 +1375,7 @@ typedef struct { unsigned short allContentSize; } LZ4IO_cFileInfo_t; -#define LZ4IO_INIT_CFILEINFO { NULL, 0ULL, 0, LZ4IO_INIT_FRAMEINFO, 1, 1, 1 } +#define LZ4IO_INIT_CFILEINFO { NULL, 0ULL, 0, LZ4IO_INIT_FRAMEINFO, 1, 1, 1 } typedef enum { LZ4IO_LZ4F_OK, LZ4IO_format_not_known, LZ4IO_not_a_file } LZ4IO_infoResult; @@ -1363,9 +1387,11 @@ static const char * LZ4IO_frameTypeNames[] = {"LZ4Frame", "LegacyFrame", "Skippa returns 0 in case it can't succesfully skip block data. Assumes SEEK_CUR after frame header. */ -static unsigned long long LZ4IO_skipBlocksData(FILE* finput, - const LZ4F_blockChecksum_t blockChecksumFlag, - const LZ4F_contentChecksum_t contentChecksumFlag) { +static unsigned long long +LZ4IO_skipBlocksData(FILE* finput, + const LZ4F_blockChecksum_t blockChecksumFlag, + const LZ4F_contentChecksum_t contentChecksumFlag) +{ unsigned char blockInfo[LZ4F_BLOCK_HEADER_SIZE]; unsigned long long totalBlocksSize = 0; for (;;) { @@ -1374,8 +1400,7 @@ static unsigned long long LZ4IO_skipBlocksData(FILE* finput, return 0; } totalBlocksSize += LZ4F_BLOCK_HEADER_SIZE; - { - const unsigned long nextCBlockSize = LZ4IO_readLE32(&blockInfo) & 0x7FFFFFFFU; + { const unsigned long nextCBlockSize = LZ4IO_readLE32(&blockInfo) & 0x7FFFFFFFU; const unsigned long nextBlock = nextCBlockSize + (blockChecksumFlag * LZ4F_BLOCK_CHECKSUM_SIZE); if (nextCBlockSize == 0) { /* Reached EndMark */ @@ -1390,11 +1415,9 @@ static unsigned long long LZ4IO_skipBlocksData(FILE* finput, } totalBlocksSize += nextBlock; /* skip to the next block */ - if (UTIL_fseek(finput, nextBlock, SEEK_CUR) != 0) { - return 0; - } - } - } + assert(nextBlock < LONG_MAX); + if (UTIL_fseek(finput, (long)nextBlock, SEEK_CUR) != 0) return 0; + } } return totalBlocksSize; } @@ -1405,7 +1428,8 @@ static unsigned long long LZ4IO_skipBlocksData(FILE* finput, This works as long as legacy block header size = magic number size. Assumes SEEK_CUR after frame header. */ -static unsigned long long LZ4IO_skipLegacyBlocksData(FILE* finput) { +static unsigned long long LZ4IO_skipLegacyBlocksData(FILE* finput) +{ unsigned char blockInfo[LZIO_LEGACY_BLOCK_HEADER_SIZE]; unsigned long long totalBlocksSize = 0; LZ4IO_STATIC_ASSERT(LZIO_LEGACY_BLOCK_HEADER_SIZE == MAGICNUMBER_SIZE); @@ -1428,14 +1452,15 @@ static unsigned long long LZ4IO_skipLegacyBlocksData(FILE* finput) { /* skip to the next block */ if (UTIL_fseek(finput, nextCBlockSize, SEEK_CUR) != 0) { return 0; - } - } - } + } } } return totalBlocksSize; } -/* buffer : must be a valid memory area of at least 4 bytes */ -const char* LZ4IO_blockTypeID(int sizeID, int blockMode, char* buffer) { +/* LZ4IO_blockTypeID: + * return human-readable block type, following command line convention + * buffer : must be a valid memory area of at least 4 bytes */ +const char* LZ4IO_blockTypeID(LZ4F_blockSizeID_t sizeID, LZ4F_blockMode_t blockMode, char buffer[4]) +{ buffer[0] = 'B'; assert(sizeID >= 4); assert(sizeID <= 7); buffer[1] = (char)(sizeID + '0'); @@ -1445,7 +1470,8 @@ const char* LZ4IO_blockTypeID(int sizeID, int blockMode, char* buffer) { } /* buffer : must be valid memory area of at least 10 bytes */ -static const char* LZ4IO_toHuman(long double size, char *buf) { +static const char* LZ4IO_toHuman(long double size, char *buf) +{ const char units[] = {"\0KMGTPEZY"}; size_t i = 0; for (; size >= 1024; i++) size /= 1024; @@ -1454,14 +1480,15 @@ static const char* LZ4IO_toHuman(long double size, char *buf) { } /* Get filename without path prefix */ -static const char* LZ4IO_baseName(const char* input_filename) { +static const char* LZ4IO_baseName(const char* input_filename) +{ const char* b = strrchr(input_filename, '/'); if (!b) b = strrchr(input_filename, '\\'); if (!b) return input_filename; return b + 1; } -/* Report frame/s information in verbose mode. +/* Report frame/s information (--list) in verbose mode (-v). * Will populate file info with fileName and frameSummary where applicable. * - TODO : * + report nb of blocks, hence max. possible decompressed size (when not reported in header) @@ -1480,11 +1507,12 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam LZ4IO_frameInfo_t frameInfo = LZ4IO_INIT_FRAMEINFO; unsigned magicNumber; /* Get MagicNumber */ - size_t nbReadBytes = fread(buffer, 1, MAGICNUMBER_SIZE, finput); - if (nbReadBytes == 0) { break; } /* EOF */ - result = LZ4IO_format_not_known; /* default result (error) */ - if (nbReadBytes != MAGICNUMBER_SIZE) - EXM_THROW(40, "Unrecognized header : Magic Number unreadable"); + { size_t const nbReadBytes = fread(buffer, 1, MAGICNUMBER_SIZE, finput); + if (nbReadBytes == 0) { break; } /* EOF */ + result = LZ4IO_format_not_known; /* default result (error) */ + if (nbReadBytes != MAGICNUMBER_SIZE) { + EXM_THROW(40, "Unrecognized header : Magic Number unreadable"); + } } magicNumber = LZ4IO_readLE32(buffer); /* Little Endian format */ if (LZ4IO_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4IO_SKIPPABLE0; /* fold skippable magic numbers */ @@ -1497,56 +1525,49 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam if (!readBytes || ferror(finput)) EXM_THROW(71, "Error reading %s", input_filename); } { size_t hSize = LZ4F_headerSize(&buffer, LZ4F_HEADER_SIZE_MIN); - if (!LZ4F_isError(hSize)) { - if (hSize > (LZ4F_HEADER_SIZE_MIN + MAGICNUMBER_SIZE)) { - /* We've already read LZ4F_HEADER_SIZE_MIN so read any extra until hSize*/ - const size_t readBytes = fread(buffer + LZ4F_HEADER_SIZE_MIN, 1, hSize - LZ4F_HEADER_SIZE_MIN, finput); - if (!readBytes || ferror(finput)) EXM_THROW(72, "Error reading %s", input_filename); - } - /* Create decompression context */ - { LZ4F_dctx* dctx; - unsigned isError = LZ4F_isError(LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION)); - if (!isError) { - isError = LZ4F_isError(LZ4F_getFrameInfo(dctx, &frameInfo.lz4FrameInfo, buffer, &hSize)); - LZ4F_freeDecompressionContext(dctx); - if (!isError) { - if ((cfinfo->frameSummary.lz4FrameInfo.blockSizeID != frameInfo.lz4FrameInfo.blockSizeID || - cfinfo->frameSummary.lz4FrameInfo.blockMode != frameInfo.lz4FrameInfo.blockMode) - && cfinfo->frameCount != 0) - cfinfo->eqBlockTypes = 0; - { const unsigned long long totalBlocksSize = LZ4IO_skipBlocksData(finput, - frameInfo.lz4FrameInfo.blockChecksumFlag, - frameInfo.lz4FrameInfo.contentChecksumFlag); - if (totalBlocksSize) { - char bTypeBuffer[5]; - LZ4IO_blockTypeID(frameInfo.lz4FrameInfo.blockSizeID, frameInfo.lz4FrameInfo.blockMode, bTypeBuffer); - DISPLAYLEVEL(3, " %6llu %14s %5s %8s", - cfinfo->frameCount + 1, - LZ4IO_frameTypeNames[frameInfo.frameType], - bTypeBuffer, - frameInfo.lz4FrameInfo.contentChecksumFlag ? "XXH32" : "-"); - if (frameInfo.lz4FrameInfo.contentSize) { - { double const ratio = (double)(totalBlocksSize + hSize) / frameInfo.lz4FrameInfo.contentSize * 100; - DISPLAYLEVEL(3, " %20llu %20llu %9.2f%%\n", - totalBlocksSize + hSize, - frameInfo.lz4FrameInfo.contentSize, - ratio); - } - /* Now we've consumed frameInfo we can use it to store the total contentSize */ - frameInfo.lz4FrameInfo.contentSize += cfinfo->frameSummary.lz4FrameInfo.contentSize; - } - else { - DISPLAYLEVEL(3, " %20llu %20s %9s \n", totalBlocksSize + hSize, "-", "-"); - cfinfo->allContentSize = 0; - } - result = LZ4IO_LZ4F_OK; + if (LZ4F_isError(hSize)) break; + if (hSize > (LZ4F_HEADER_SIZE_MIN + MAGICNUMBER_SIZE)) { + /* We've already read LZ4F_HEADER_SIZE_MIN so read any extra until hSize*/ + const size_t readBytes = fread(buffer + LZ4F_HEADER_SIZE_MIN, 1, hSize - LZ4F_HEADER_SIZE_MIN, finput); + if (!readBytes || ferror(finput)) EXM_THROW(72, "Error reading %s", input_filename); + } + /* Create decompression context */ + { LZ4F_dctx* dctx; + if ( LZ4F_isError(LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION)) ) break; + { unsigned const frameInfoError = LZ4F_isError(LZ4F_getFrameInfo(dctx, &frameInfo.lz4FrameInfo, buffer, &hSize)); + LZ4F_freeDecompressionContext(dctx); + if (frameInfoError) break; + if ((cfinfo->frameSummary.lz4FrameInfo.blockSizeID != frameInfo.lz4FrameInfo.blockSizeID || + cfinfo->frameSummary.lz4FrameInfo.blockMode != frameInfo.lz4FrameInfo.blockMode) + && cfinfo->frameCount != 0) + cfinfo->eqBlockTypes = 0; + { const unsigned long long totalBlocksSize = LZ4IO_skipBlocksData(finput, + frameInfo.lz4FrameInfo.blockChecksumFlag, + frameInfo.lz4FrameInfo.contentChecksumFlag); + if (totalBlocksSize) { + char bTypeBuffer[5]; + LZ4IO_blockTypeID(frameInfo.lz4FrameInfo.blockSizeID, frameInfo.lz4FrameInfo.blockMode, bTypeBuffer); + DISPLAYLEVEL(3, " %6llu %14s %5s %8s", + cfinfo->frameCount + 1, + LZ4IO_frameTypeNames[frameInfo.frameType], + bTypeBuffer, + frameInfo.lz4FrameInfo.contentChecksumFlag ? "XXH32" : "-"); + if (frameInfo.lz4FrameInfo.contentSize) { + { double const ratio = (double)(totalBlocksSize + hSize) / frameInfo.lz4FrameInfo.contentSize * 100; + DISPLAYLEVEL(3, " %20llu %20llu %9.2f%%\n", + totalBlocksSize + hSize, + frameInfo.lz4FrameInfo.contentSize, + ratio); } + /* Now we've consumed frameInfo we can use it to store the total contentSize */ + frameInfo.lz4FrameInfo.contentSize += cfinfo->frameSummary.lz4FrameInfo.contentSize; } - } - } - } - } - } + else { + DISPLAYLEVEL(3, " %20llu %20s %9s \n", totalBlocksSize + hSize, "-", "-"); + cfinfo->allContentSize = 0; + } + result = LZ4IO_LZ4F_OK; + } } } } } break; case LEGACY_MAGICNUMBER: frameInfo.frameType = legacyFrame; @@ -1569,7 +1590,7 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam if (cfinfo->frameSummary.frameType != skippableFrame && cfinfo->frameCount != 0) cfinfo->eqFrameTypes = 0; cfinfo->eqBlockTypes = 0; cfinfo->allContentSize = 0; - { nbReadBytes = fread(buffer, 1, 4, finput); + { size_t const nbReadBytes = fread(buffer, 1, 4, finput); if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable"); } @@ -1594,12 +1615,10 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam } break; } - if (result != LZ4IO_LZ4F_OK) { - break; - } + if (result != LZ4IO_LZ4F_OK) break; cfinfo->frameSummary = frameInfo; cfinfo->frameCount++; - } + } /* while (!feof(finput)) */ fclose(finput); return result; } |