diff options
| -rw-r--r-- | programs/lz4.1 | 24 | ||||
| -rw-r--r-- | programs/lz4io.c | 176 | ||||
| -rw-r--r-- | programs/lz4io.h | 3 | ||||
| -rw-r--r-- | tests/Makefile | 1 |
4 files changed, 111 insertions, 93 deletions
diff --git a/programs/lz4.1 b/programs/lz4.1 index 1576e45..eb82b68 100644 --- a/programs/lz4.1 +++ b/programs/lz4.1 @@ -1,5 +1,5 @@ . -.TH "LZ4" "1" "April 2019" "lz4 1.9.0" "User Commands" +.TH "LZ4" "1" "April 2019" "lz4 1.9.1" "User Commands" . .SH "NAME" \fBlz4\fR \- lz4, unlz4, lz4cat \- Compress or decompress \.lz4 files @@ -23,9 +23,6 @@ When writing scripts that need to decompress files, it is recommended to always \fBlz4\fR supports a command line syntax similar \fIbut not identical\fR to \fBgzip(1)\fR\. Differences are : . .IP "\(bu" 4 -\fBlz4\fR preserves original files -. -.IP "\(bu" 4 \fBlz4\fR compresses a single file by default (see \fB\-m\fR for multiple files) . .IP "\(bu" 4 @@ -35,19 +32,16 @@ When writing scripts that need to decompress files, it is recommended to always \fBlz4 file\.lz4\fR will default to decompression (use \fB\-z\fR to force compression) . .IP "\(bu" 4 -\fBlz4\fR shows real\-time notification statistics during compression or decompression of a single file (use \fB\-q\fR to silence them) -. -.IP "\(bu" 4 -If no destination name is provided, result is sent to \fBstdout\fR \fIexcept if stdout is the console\fR\. +\fBlz4\fR preserves original files . .IP "\(bu" 4 -If no destination name is provided, \fBand\fR if \fBstdout\fR is the console, \fBfile\fR is compressed into \fBfile\.lz4\fR\. +\fBlz4\fR shows real\-time notification statistics during compression or decompression of a single file (use \fB\-q\fR to silence them) . .IP "\(bu" 4 -As a consequence of previous rules, note the following example : \fBlz4 file | consumer\fR sends compressed data to \fBconsumer\fR through \fBstdout\fR, hence it does \fInot\fR create \fBfile\.lz4\fR\. +When no destination is specified, result is sent on implicit output, which depends on \fBstdout\fR status\. When \fBstdout\fR \fIis Not the console\fR, it becomes the implicit output\. Otherwise, if \fBstdout\fR is the console, the implicit output is \fBfilename\.lz4\fR\. . .IP "\(bu" 4 -Another consequence of those rules is that to run \fBlz4\fR under \fBnohup\fR, you should provide a destination file: \fBnohup lz4 file file\.lz4\fR, because \fBnohup\fR writes the specified command\'s output to a file\. +It is considered bad practice to rely on implicit output in scripts\. because the script\'s environment may change\. Always use explicit output in scripts\. \fB\-c\fR ensures that output will be \fBstdout\fR\. Conversely, providing a destination name, or using \fB\-m\fR ensures that the output will be either the specified name, or \fBfilename\.lz4\fR respectively\. . .IP "" 0 . @@ -55,7 +49,7 @@ Another consequence of those rules is that to run \fBlz4\fR under \fBnohup\fR, y Default behaviors can be modified by opt\-in commands, detailed below\. . .IP "\(bu" 4 -\fBlz4 \-m\fR makes it possible to provide multiple input filenames, which will be compressed into files using suffix \fB\.lz4\fR\. Progress notifications are also disabled by default (use \fB\-v\fR to enable them)\. This mode has a behavior which more closely mimics \fBgzip\fR command line, with the main remaining difference being that source files are preserved by default\. +\fBlz4 \-m\fR makes it possible to provide multiple input filenames, which will be compressed into files using suffix \fB\.lz4\fR\. Progress notifications become disabled by default (use \fB\-v\fR to enable them)\. This mode has a behavior which more closely mimics \fBgzip\fR command line, with the main remaining difference being that source files are preserved by default\. . .IP "\(bu" 4 Similarly, \fBlz4 \-m \-d\fR can decompress multiple \fB*\.lz4\fR files\. @@ -111,6 +105,10 @@ Test the integrity of compressed \fB\.lz4\fR files\. The decompressed data is di \fB\-b#\fR Benchmark mode, using \fB#\fR compression level\. . +.TP +\fB\-\-list\fR +List mode\. Lists information about \.lz4 files\. Useful if compressed with \-\-content\-size flag\. +. .SS "Operation modifiers" . .TP @@ -145,7 +143,7 @@ Force write to standard output, even if it is the console\. . .TP \fB\-m\fR \fB\-\-multiple\fR -Multiple input files\. Compressed file names will be appended a \fB\.lz4\fR suffix\. This mode also reduces notification level\. \fBlz4 \-m\fR has a behavior equivalent to \fBgzip \-k\fR (it preserves source files by default)\. +Multiple input files\. Compressed file names will be appended a \fB\.lz4\fR suffix\. This mode also reduces notification level\. Can also be used to list multiple files\. \fBlz4 \-m\fR has a behavior equivalent to \fBgzip \-k\fR (it preserves source files by default)\. . .TP \fB\-r\fR diff --git a/programs/lz4io.c b/programs/lz4io.c index 1e6efe1..e3eed93 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -1275,87 +1275,105 @@ typedef struct { LZ4F_frameInfo_t frameInfo; const char* fileName; unsigned long long fileSize; -} LZ4F_compFileInfo_t; - -#define LZ4F_INIT_FILEINFO { LZ4F_INIT_FRAMEINFO, NULL, 0ULL } - - -static int LZ4IO_getCompressedFileInfo(const char* input_filename, LZ4F_compFileInfo_t* cfinfo){ - const char *b, - *e; - char *t; - stat_t statbuf; - size_t readSize = LZ4F_HEADER_SIZE_MAX; - LZ4F_errorCode_t errorCode; - dRess_t ress; - /* Open file */ - FILE* const finput = LZ4IO_openSrcFile(input_filename); - if (finput==NULL) return 1; - - /* Get file size */ - if (!UTIL_getFileStat(input_filename, &statbuf)){ - EXM_THROW(60, "Can't stat file : %s", input_filename); - } - - cfinfo->fileSize = (unsigned long long)statbuf.st_size; - - /* Get basename without extension */ - b = strrchr(input_filename, '/'); - if (!b){ - b = strrchr(input_filename, '\\'); - } - if (b && b != input_filename){ - b++; - } else{ - b=input_filename; - } - e = strrchr(b, '.'); - - /* Allocate Memory */ - t = (char*)malloc( (size_t)(e-b+1) * sizeof(char)); - ress.srcBuffer = malloc(LZ4IO_dBufferSize); - if (!t || !ress.srcBuffer) - EXM_THROW(21, "Allocation error : not enough memory"); - strncpy(t, b, (e-b)); - t[e-b] = '\0'; - cfinfo->fileName = t; - - /* init */ - errorCode = LZ4F_createDecompressionContext(&ress.dCtx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) EXM_THROW(60, "Can't create LZ4F context : %s", LZ4F_getErrorName(errorCode)); - - if (!fread(ress.srcBuffer, readSize, 1, finput)){ - EXM_THROW(30, "Error reading %s ", input_filename); - } - LZ4F_getFrameInfo(ress.dCtx, &cfinfo->frameInfo, ress.srcBuffer, &readSize); - - /* Close input/free resources */ - fclose(finput); - free(ress.srcBuffer); - return 0; -} +} LZ4IO_cFileInfo_t; + +#define LZ4IO_INIT_CFILEINFO { LZ4F_INIT_FRAMEINFO, NULL, 0ULL } + + +/* This function is limited, + * it only works fine for a file consisting of a single valid frame. + * It will / may stop program execution if a single filename is wrong. + * It will not look at content beyond first frame header. + * + * Things to improve : + * - continue execution after an error, just report an error code, keep all memory clean + * - check the entire file for additional content after first frame + * + combine results from multiple frames, give total + * - Optional : + * + report nb of blocks, hence max. possible decompressed size (when not reported in header) + * + report block type (B4D, B7I, etc.) + */ +static int +LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filename) +{ + /* Get file size */ + cfinfo->fileSize = UTIL_getFileSize(input_filename); /* returns 0 if cannot read information */ -int LZ4IO_displayCompressedFilesInfo(const char** inFileNames, const size_t ifnIdx){ - size_t idx; - int op_result=0; - double ratio; - DISPLAY("%16s\t%-20s\t%-20s\t%-10s\t%s\n","BlockChecksumFlag","Compressed", "Uncompressed", "Ratio", "Filename"); - for(idx=0; idx<ifnIdx; idx++){ - /* Get file info */ - LZ4F_compFileInfo_t cfinfo = LZ4F_INIT_FILEINFO; - op_result=LZ4IO_getCompressedFileInfo(inFileNames[idx], &cfinfo); - if (op_result != 0){ - DISPLAYLEVEL(1, "Failed to get frame info for file %s\n", inFileNames[idx]); - /* Don't bother trying to process any other file */ - break; + /* Get filename without path prefix */ + { const char* b = strrchr(input_filename, '/'); + if (!b) { + b = strrchr(input_filename, '\\'); + } + if (b && b != input_filename) { + b++; + } else { + b = input_filename; + } + cfinfo->fileName = b; } - if(cfinfo.frameInfo.contentSize){ - ratio = (double)cfinfo.fileSize / cfinfo.frameInfo.contentSize; - DISPLAY("%-16d\t%-20llu\t%-20llu\t%-8.4f\t%s\n",cfinfo.frameInfo.blockChecksumFlag,cfinfo.fileSize,cfinfo.frameInfo.contentSize, ratio, cfinfo.fileName); + + /* Read file and extract header */ + { size_t readSize = LZ4F_HEADER_SIZE_MAX; + void* buffer = malloc(readSize); + LZ4F_dctx* dctx; + + if (!buffer) EXM_THROW(21, "Allocation error : not enough memory"); + { LZ4F_errorCode_t const errorCode = + LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION); + if (LZ4F_isError(errorCode)) + EXM_THROW(60, "Can't create LZ4F context : %s", + LZ4F_getErrorName(errorCode)); + } + + { FILE* const finput = LZ4IO_openSrcFile(input_filename); + if (finput==NULL) return 1; + if (!fread(buffer, readSize, 1, finput)) { + EXM_THROW(30, "Error reading %s ", input_filename); + } + fclose(finput); + } + + { LZ4F_errorCode_t const errorCode = + LZ4F_getFrameInfo(dctx, &cfinfo->frameInfo, buffer, &readSize); + if (LZ4F_isError(errorCode)) + EXM_THROW(60, "Cannot interpret LZ4 frame : %s", + LZ4F_getErrorName(errorCode)); + } + + /* clean */ + free(buffer); + LZ4F_freeDecompressionContext(dctx); } - else{ - DISPLAY("%-16d\t%-20llu\t%-20s\t%-10s\t%s\n",cfinfo.frameInfo.blockChecksumFlag,cfinfo.fileSize, "-", "-", cfinfo.fileName); + + return 0; +} + +int LZ4IO_displayCompressedFilesInfo(const char** inFileNames, const size_t ifnIdx) +{ + size_t idx; + + DISPLAY("%16s\t%-20s\t%-20s\t%-10s\t%s\n", + "BlockChecksumFlag","Compressed", "Uncompressed", "Ratio", "Filename"); + + for (idx=0; idx<ifnIdx; idx++) { + /* Get file info */ + LZ4IO_cFileInfo_t cfinfo = LZ4IO_INIT_CFILEINFO; + int const op_result = LZ4IO_getCompressedFileInfo(&cfinfo, inFileNames[idx]); + if (op_result != 0) { + DISPLAYLEVEL(1, "Failed to get frame info for file %s\n", inFileNames[idx]); + /* Don't bother processing any more file */ + return 1; + } + if (cfinfo.frameInfo.contentSize) { + double const ratio = (double)cfinfo.fileSize / cfinfo.frameInfo.contentSize; + DISPLAY("%-16d\t%-20llu\t%-20llu\t%-8.4f\t%s\n", + cfinfo.frameInfo.blockChecksumFlag, cfinfo.fileSize, + cfinfo.frameInfo.contentSize, ratio, cfinfo.fileName); + } else { + DISPLAY("%-16d\t%-20llu\t%-20s\t%-10s\t%s\n", + cfinfo.frameInfo.blockChecksumFlag, cfinfo.fileSize, + "-", "-", cfinfo.fileName); + } } - } - return op_result; + return 0; } diff --git a/programs/lz4io.h b/programs/lz4io.h index 7dba954..ff96d6d 100644 --- a/programs/lz4io.h +++ b/programs/lz4io.h @@ -124,7 +124,8 @@ void LZ4IO_setRemoveSrcFile(LZ4IO_prefs_t* const prefs, unsigned flag); void LZ4IO_favorDecSpeed(LZ4IO_prefs_t* const prefs, int favor); -/* implement --list */ +/* implement --list + * @return 0 on success, 1 on error */ int LZ4IO_displayCompressedFilesInfo(const char** inFileNames,const size_t ifnIdx); diff --git a/tests/Makefile b/tests/Makefile index ddc6d1e..003e8bd 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -305,6 +305,7 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat test ! -f ./-z $(DIFF) -q tmp-tlb-hw tmp-tlb4 $(LZ4) -f tmp-tlb-hw + $(LZ4) --list tmp-tlb-hw.lz4 # test --list on valid single-frame file cat tmp-tlb-hw >> tmp-tlb-hw.lz4 $(LZ4) -f tmp-tlb-hw.lz4 # uncompress valid frame followed by invalid data $(LZ4) -BX tmp-tlb-hw -c -q | $(LZ4) -tv # test block checksum |