diff options
| -rw-r--r-- | programs/Makefile | 21 | ||||
| -rw-r--r-- | programs/lz4cli.c | 5 | ||||
| -rw-r--r-- | programs/lz4io.c | 270 |
3 files changed, 155 insertions, 141 deletions
diff --git a/programs/Makefile b/programs/Makefile index 7bbe060..6eec96e 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -191,7 +191,19 @@ test-lz4-frame-concatenation: lz4 datagen @rm *.test @echo frame concatenation test completed -test-lz4: lz4 datagen test-lz4-sparse test-lz4-contentSize test-lz4-frame-concatenation +test-lz4-multiple: lz4 datagen + @echo ---- test multiple files ---- + @./datagen -s1 > tmp1 2> $(VOID) + @./datagen -s2 -g100K > tmp2 2> $(VOID) + @./datagen -s3 -g1M > tmp3 2> $(VOID) + ./lz4 -f -m tmp* + ls -ls tmp* + rm tmp1 tmp2 tmp3 + ./lz4 -df -m *.lz4 + ls -ls tmp* + @rm tmp* + +test-lz4: lz4 datagen test-lz4-multiple test-lz4-sparse test-lz4-contentSize test-lz4-frame-concatenation @echo ---- test lz4 basic compression/decompression ---- ./datagen -g0 | ./lz4 -v | ./lz4 -t ./datagen -g16KB | ./lz4 -9 | ./lz4 -t @@ -202,13 +214,6 @@ test-lz4: lz4 datagen test-lz4-sparse test-lz4-contentSize test-lz4-frame-concat ./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -t ./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -qt ./datagen -g6GB | ./lz4 -vq9BD | ./lz4 -qt - @echo ---- test multiple input files ---- - @./datagen -s1 > file1 - @./datagen -s2 > file2 - @./datagen -s3 > file3 - ./lz4 -f -m file1 file2 file3 - ls -l file* - @rm file1 file2 file3 file1.lz4 file2.lz4 file3.lz4 @echo ---- test pass-through ---- ./datagen | ./lz4 -tf diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 8d7d5ca..d77ef5b 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -102,7 +102,7 @@ ***************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static unsigned displayLevel = 2; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */ +static unsigned displayLevel = 2; /* 0 : no display ; 1: errors only ; 2 : downgradable normal ; 3 : non-downgradable normal; 4 : + information */ /************************************** @@ -521,8 +521,9 @@ int main(int argc, char** argv) /* Check if output is defined as console; trigger an error in this case */ if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) badusage(); - /* No warning message in pure pipe mode (stdin + stdout) */ + /* Downgrade notification level in pure pipe mode (stdin + stdout) and multiple file mode */ if (!strcmp(input_filename, stdinmark) && !strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1; + if ((multiple_inputs) && (displayLevel==2)) displayLevel=1; /* IO Stream/File */ diff --git a/programs/lz4io.c b/programs/lz4io.c index 6ca7ee7..ba2a6e8 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -159,7 +159,7 @@ static const int maxBlockSizeID = 7; #define EXTENDED_ARGUMENTS #define EXTENDED_HELP #define EXTENDED_FORMAT -#define DEFAULT_DECOMPRESSOR LZ4IO_decompressFile +#define DEFAULT_DECOMPRESSOR LZ4IO_decompressLZ4F /* ************************************************** */ @@ -421,7 +421,7 @@ static cRess_t LZ4IO_createCResources(void) /* Allocate Memory */ ress.srcBuffer = malloc(blockSize); ress.srcBufferSize = blockSize; - ress.dstBufferSize = LZ4F_compressBound(blockSize, NULL); /* cover worst case */ + ress.dstBufferSize = LZ4F_compressFrameBound(blockSize, NULL); /* cover worst case */ ress.dstBuffer = malloc(ress.dstBufferSize); if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory"); @@ -438,7 +438,7 @@ static void LZ4IO_freeCResources(cRess_t ress) } -static int LZ4IO_compressJob(cRess_t ress, const char* srcFileName, const char* dstFileName, int compressionLevel) +static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, const char* dstFileName, int compressionLevel) { unsigned long long filesize = 0; unsigned long long compressedfilesize = 0; @@ -476,44 +476,63 @@ static int LZ4IO_compressJob(cRess_t ress, const char* srcFileName, const char* DISPLAYLEVEL(3, "Warning : cannot determine uncompressed frame content size \n"); } - /* Write Archive Header */ - headerSize = LZ4F_compressBegin(ctx, dstBuffer, dstBufferSize, &prefs); - if (LZ4F_isError(headerSize)) EXM_THROW(32, "File header generation failed : %s", LZ4F_getErrorName(headerSize)); - sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); - if (sizeCheck!=headerSize) EXM_THROW(33, "Write error : cannot write header"); - compressedfilesize += headerSize; - /* read first block */ - readSize = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile); + readSize = fread(srcBuffer, (size_t)1, blockSize, srcFile); filesize += readSize; - /* Main Loop */ - while (readSize>0) + /* single-block file */ + if (readSize < blockSize) { - size_t outSize; - - /* Compress Block */ - outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL); - if (LZ4F_isError(outSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(outSize)); - compressedfilesize += outSize; + /* Compress in single pass */ + size_t cSize = LZ4F_compressFrame(dstBuffer, dstBufferSize, srcBuffer, readSize, &prefs); + if (LZ4F_isError(cSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(cSize)); + compressedfilesize += cSize; DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100); /* Write Block */ - sizeCheck = fwrite(dstBuffer, 1, outSize, dstFile); - if (sizeCheck!=outSize) EXM_THROW(35, "Write error : cannot write compressed block"); - - /* Read next block */ - readSize = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile); - filesize += readSize; + sizeCheck = fwrite(dstBuffer, 1, cSize, dstFile); + if (sizeCheck!=cSize) EXM_THROW(35, "Write error : cannot write compressed block"); } - /* End of Stream mark */ - headerSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL); - if (LZ4F_isError(headerSize)) EXM_THROW(36, "End of file generation failed : %s", LZ4F_getErrorName(headerSize)); + else + + /* multiple-blocks file */ + { + /* Write Archive Header */ + headerSize = LZ4F_compressBegin(ctx, dstBuffer, dstBufferSize, &prefs); + if (LZ4F_isError(headerSize)) EXM_THROW(32, "File header generation failed : %s", LZ4F_getErrorName(headerSize)); + sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); + if (sizeCheck!=headerSize) EXM_THROW(33, "Write error : cannot write header"); + compressedfilesize += headerSize; + + /* Main Loop */ + while (readSize>0) + { + size_t outSize; + + /* Compress Block */ + outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL); + if (LZ4F_isError(outSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(outSize)); + compressedfilesize += outSize; + DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100); + + /* Write Block */ + sizeCheck = fwrite(dstBuffer, 1, outSize, dstFile); + if (sizeCheck!=outSize) EXM_THROW(35, "Write error : cannot write compressed block"); - sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); - if (sizeCheck!=headerSize) EXM_THROW(37, "Write error : cannot write end of stream"); - compressedfilesize += headerSize; + /* Read next block */ + readSize = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile); + filesize += readSize; + } + + /* End of Stream mark */ + headerSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL); + if (LZ4F_isError(headerSize)) EXM_THROW(36, "End of file generation failed : %s", LZ4F_getErrorName(headerSize)); + + sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); + if (sizeCheck!=headerSize) EXM_THROW(37, "Write error : cannot write end of stream"); + compressedfilesize += headerSize; + } /* Release files */ fclose (srcFile); @@ -521,14 +540,10 @@ static int LZ4IO_compressJob(cRess_t ress, const char* srcFileName, const char* /* Final Status */ DISPLAYLEVEL(2, "\r%79s\r", ""); - if (filesize == 0) - { - DISPLAYLEVEL(2, "Empty input\n"); - } - else { DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", - (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); + (unsigned long long) filesize, (unsigned long long) compressedfilesize, + (double)compressedfilesize/(filesize + !filesize)*100); /* avoid division by zero */ } return 0; @@ -546,7 +561,7 @@ int LZ4IO_compressFilename(const char* srcFileName, const char* dstFileName, int ress = LZ4IO_createCResources(); /* Compress File */ - issueWithSrcFile += LZ4IO_compressJob(ress, srcFileName, dstFileName, compressionLevel); + issueWithSrcFile += LZ4IO_compressFilename_extRess(ress, srcFileName, dstFileName, compressionLevel); /* Free resources */ LZ4IO_freeCResources(ress); @@ -583,7 +598,7 @@ int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, strcpy(outFileName, inFileNamesTable[i]); strcat(outFileName, suffix); - missing_files += LZ4IO_compressJob(ress, inFileNamesTable[i], outFileName, compressionLevel); + missing_files += LZ4IO_compressFilename_extRess(ress, inFileNamesTable[i], outFileName, compressionLevel); } /* Close & Free */ @@ -751,16 +766,44 @@ static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput) typedef struct { - FILE* sFile; - FILE* dFile; void* srcBuffer; size_t srcBufferSize; void* dstBuffer; size_t dstBufferSize; LZ4F_decompressionContext_t dCtx; -} decompressionJob_t; +} dRess_t; + +static const size_t LZ4IO_dBufferSize = 64 KB; -static unsigned long long LZ4IO_decompressJob(decompressionJob_t ress) +static dRess_t LZ4IO_createDResources(void) +{ + dRess_t ress; + LZ4F_errorCode_t errorCode; + + /* init */ + errorCode = LZ4F_createDecompressionContext(&ress.dCtx, LZ4F_VERSION); + if (LZ4F_isError(errorCode)) EXM_THROW(60, "Can't create LZ4F context : %s", LZ4F_getErrorName(errorCode)); + + /* Allocate Memory */ + ress.srcBufferSize = LZ4IO_dBufferSize; + ress.srcBuffer = malloc(ress.srcBufferSize); + ress.dstBufferSize = LZ4IO_dBufferSize; + ress.dstBuffer = malloc(ress.dstBufferSize); + if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory"); + + return ress; +} + +static void LZ4IO_freeDResources(dRess_t ress) +{ + LZ4F_errorCode_t errorCode = LZ4F_freeDecompressionContext(ress.dCtx); + if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); + free(ress.srcBuffer); + free(ress.dstBuffer); +} + + +static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE* dstFile) { unsigned long long filesize = 0; LZ4F_errorCode_t errorCode; @@ -782,7 +825,7 @@ static unsigned long long LZ4IO_decompressJob(decompressionJob_t ress) size_t pos = 0; /* Read input */ - readSize = fread(ress.srcBuffer, 1, ress.srcBufferSize, ress.sFile); + readSize = fread(ress.srcBuffer, 1, ress.srcBufferSize, srcFile); if (!readSize) break; /* empty file or stream */ while (pos < readSize) @@ -799,69 +842,26 @@ static unsigned long long LZ4IO_decompressJob(decompressionJob_t ress) /* Write Block */ filesize += decodedBytes; DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20)); - storedSkips = LZ4IO_fwriteSparse(ress.dFile, ress.dstBuffer, decodedBytes, storedSkips); + storedSkips = LZ4IO_fwriteSparse(dstFile, ress.dstBuffer, decodedBytes, storedSkips); } } } - LZ4IO_fwriteSparseEnd(ress.dFile, storedSkips); - - return filesize; -} - - -static const size_t LZ4IO_dBufferSize = 64 KB; - -static unsigned long long LZ4IO_decompressFile(FILE* finput, FILE* foutput) -{ - unsigned long long filesize = 0; - void* inBuff; - void* outBuff; - const size_t inBuffSize = LZ4IO_dBufferSize; - const size_t outBuffSize = LZ4IO_dBufferSize; - LZ4F_decompressionContext_t dCtx; - LZ4F_errorCode_t errorCode; - decompressionJob_t ress; - - /* init */ - errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) EXM_THROW(60, "Can't create context : %s", LZ4F_getErrorName(errorCode)); - - /* Allocate Memory */ - inBuff = malloc(inBuffSize); - outBuff = malloc(outBuffSize); - if (!inBuff || !outBuff) EXM_THROW(61, "Allocation error : not enough memory"); - - /* Decompression Job */ - ress.dCtx = dCtx; - ress.dFile = foutput; - ress.dstBuffer = outBuff; - ress.dstBufferSize = outBuffSize; - ress.sFile = finput; - ress.srcBuffer = inBuff; - ress.srcBufferSize = inBuffSize; - - filesize = LZ4IO_decompressJob(ress); - - /* Free */ - free(inBuff); - free(outBuff); - errorCode = LZ4F_freeDecompressionContext(dCtx); - if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); + LZ4IO_fwriteSparseEnd(dstFile, storedSkips); return filesize; } -static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigned char U32store[MAGICNUMBER_SIZE]) +static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigned char MNstore[MAGICNUMBER_SIZE]) { void* buffer = malloc(64 KB); size_t read = 1, sizeCheck; unsigned long long total = MAGICNUMBER_SIZE; unsigned storedSkips = 0; - sizeCheck = fwrite(U32store, 1, MAGICNUMBER_SIZE, foutput); - if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(50, "Pass-through error at start"); + sizeCheck = fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput); + if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(50, "Pass-through write error"); while (read) { @@ -877,9 +877,9 @@ static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigne #define ENDOFSTREAM ((unsigned long long)-1) -static unsigned long long selectDecoder( FILE* finput, FILE* foutput) +static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutput) { - unsigned char U32store[MAGICNUMBER_SIZE]; + unsigned char MNstore[MAGICNUMBER_SIZE]; unsigned magicNumber, size; int errorNb; size_t nbReadBytes; @@ -896,34 +896,34 @@ static unsigned long long selectDecoder( FILE* finput, FILE* foutput) } else { - nbReadBytes = fread(U32store, 1, MAGICNUMBER_SIZE, finput); + nbReadBytes = fread(MNstore, 1, MAGICNUMBER_SIZE, finput); if (nbReadBytes==0) return ENDOFSTREAM; /* EOF */ if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(40, "Unrecognized header : Magic Number unreadable"); - magicNumber = LZ4IO_readLE32(U32store); /* Little Endian format */ + magicNumber = LZ4IO_readLE32(MNstore); /* Little Endian format */ } if (LZ4IO_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4IO_SKIPPABLE0; /* fold skippable magic numbers */ switch(magicNumber) { case LZ4IO_MAGICNUMBER: - return DEFAULT_DECOMPRESSOR(finput, foutput); + return LZ4IO_decompressLZ4F(ress, finput, foutput); case LEGACY_MAGICNUMBER: DISPLAYLEVEL(4, "Detected : Legacy format \n"); return LZ4IO_decodeLegacyStream(finput, foutput); case LZ4IO_SKIPPABLE0: DISPLAYLEVEL(4, "Skipping detected skippable area \n"); - nbReadBytes = fread(U32store, 1, 4, finput); + nbReadBytes = fread(MNstore, 1, 4, finput); if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable"); - size = LZ4IO_readLE32(U32store); /* Little Endian format */ + size = LZ4IO_readLE32(MNstore); /* Little Endian format */ errorNb = fseek(finput, size, SEEK_CUR); if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area"); - return selectDecoder(finput, foutput); + return selectDecoder(ress, finput, foutput); EXTENDED_FORMAT; default: if (nbCalls == 1) /* just started */ { if (g_overwrite) - return LZ4IO_passThrough(finput, foutput, U32store); + return LZ4IO_passThrough(finput, foutput, MNstore); EXM_THROW(44,"Unrecognized header : file cannot be decoded"); /* Wrong magic number at the beginning of 1st stream */ } DISPLAYLEVEL(2, "Stream followed by unrecognized data\n"); @@ -932,18 +932,16 @@ static unsigned long long selectDecoder( FILE* finput, FILE* foutput) } -int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename) +static int LZ4IO_decompressFile_extRess(dRess_t ress, const char* input_filename, const char* output_filename) { unsigned long long filesize = 0, decodedSize=0; FILE* finput; FILE* foutput; - clock_t start, end; /* Init */ - start = clock(); if (LZ4IO_getFiles(input_filename, output_filename, &finput, &foutput)) - EXM_THROW(50, "File error"); + return 1; /* sparse file */ if (g_sparseFileSupport) { SET_SPARSE_FILE_MODE(foutput); } @@ -951,67 +949,77 @@ int LZ4IO_decompressFilename(const char* input_filename, const char* output_file /* Loop over multiple streams */ do { - decodedSize = selectDecoder(finput, foutput); + decodedSize = selectDecoder(ress, finput, foutput); if (decodedSize != ENDOFSTREAM) filesize += decodedSize; } while (decodedSize != ENDOFSTREAM); /* Final Status */ - end = clock(); DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize); - if (end==start) end=start+1; - { - double seconds = (double)(end - start)/CLOCKS_PER_SEC; - DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); - } /* Close */ fclose(finput); fclose(foutput); - /* Error status = OK */ return 0; } +int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename) +{ + dRess_t ress; + clock_t start, end; + int missingFiles = 0; + + start = clock(); + + ress = LZ4IO_createDResources(); + missingFiles += LZ4IO_decompressFile_extRess(ress, input_filename, output_filename); + LZ4IO_freeDResources(ress); + + end = clock(); + if (end==start) end=start+1; + { + double seconds = (double)(end - start)/CLOCKS_PER_SEC; + DISPLAYLEVEL(4, "Done in %.2f sec \n", seconds); + } + + return missingFiles; +} + + int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix) { int i; - int skipped_files = 0; - int missing_files = 0; + int skippedFiles = 0; + int missingFiles = 0; char* outFileName = (char*)malloc(FNSPACE); size_t ofnSize = FNSPACE; const size_t suffixSize = strlen(suffix); char* ifnSuffix = (char*)malloc(suffixSize + 1); + dRess_t ress; + + ress = LZ4IO_createDResources(); for (i=0; i<ifntSize; i++) { - size_t ifnSize; - FILE* ifp = fopen(inFileNamesTable[i], "r"); - if (ifp == NULL) - { - DISPLAYLEVEL(2, "Unable to access file for processing: %s\n", inFileNamesTable[i]); - missing_files++; - continue; - } - fclose(ifp); - ifnSize = strlen(inFileNamesTable[i]); + size_t ifnSize = strlen(inFileNamesTable[i]); strcpy(ifnSuffix, inFileNamesTable[i] + ifnSize - suffixSize); if (ofnSize <= ifnSize-suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); } if (ifnSize <= suffixSize || strcmp(ifnSuffix, suffix) != 0) { DISPLAYLEVEL(2, "File extension doesn't match expected LZ4_EXTENSION (%4s); will not process file: %s\n", suffix, inFileNamesTable[i]); - skipped_files++; + skippedFiles++; continue; } memcpy(outFileName, inFileNamesTable[i], ifnSize - suffixSize); outFileName[ifnSize-suffixSize] = '\0'; - LZ4IO_decompressFilename(inFileNamesTable[i], outFileName); + + missingFiles += LZ4IO_decompressFile_extRess(ress, inFileNamesTable[i], outFileName); } + + LZ4IO_freeDResources(ress); free(outFileName); - free(ifnSuffix); - if (skipped_files > 0) return 1; - if (missing_files > 0) return 1; - return 0; + return missingFiles + skippedFiles; } |