From 9a7658070aeca2c0a6c68e5025a80f9e4b847ad7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Mon, 24 Aug 2020 06:23:51 +0200 Subject: Use fstat() to determine file size. This allows us to get the file size even when the input file is passed via stdin. This fixes `--content-size` not working in situations like $ lz4 -v --content-size < /tmp/test > /tmp/test.lz4 Warning : cannot determine input content size With this change, it works. Also helps with #904. --- programs/lz4io.c | 4 ++-- programs/util.h | 27 ++++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index 7926b20..0f3507e 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -680,7 +680,7 @@ LZ4IO_compressFilename_extRess(LZ4IO_prefs_t* const io_prefs, cRess_t ress, prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)io_prefs->streamChecksum; prefs.favorDecSpeed = io_prefs->favorDecSpeed; if (io_prefs->contentSizeFlag) { - U64 const fileSize = UTIL_getFileSize(srcFileName); + U64 const fileSize = UTIL_getOpenFileSize(srcFile); prefs.frameInfo.contentSize = fileSize; /* == 0 if input == stdin */ if (fileSize==0) DISPLAYLEVEL(3, "Warning : cannot determine input content size \n"); @@ -1472,7 +1472,7 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam LZ4IO_infoResult result = LZ4IO_format_not_known; /* default result (error) */ unsigned char buffer[LZ4F_HEADER_SIZE_MAX]; FILE* const finput = LZ4IO_openSrcFile(input_filename); - cfinfo->fileSize = UTIL_getFileSize(input_filename); + cfinfo->fileSize = (finput == NULL) ? 0 : UTIL_getOpenFileSize(finput); while (!feof(finput)) { LZ4IO_frameInfo_t frameInfo = LZ4IO_INIT_FRAMEINFO; diff --git a/programs/util.h b/programs/util.h index 8e361ca..4b2c09a 100644 --- a/programs/util.h +++ b/programs/util.h @@ -33,7 +33,7 @@ extern "C" { #include /* size_t, ptrdiff_t */ #include /* malloc */ #include /* strlen, strncpy */ -#include /* fprintf */ +#include /* fprintf, fileno */ #include #include /* stat, utime */ #include /* stat */ @@ -357,6 +357,31 @@ UTIL_STATIC U32 UTIL_isDirectory(const char* infilename) } +UTIL_STATIC U64 UTIL_getOpenFileSize(FILE* file) +{ + int r; + int fd = fileno(file); + if (fd < 0) { + perror("fileno"); + exit(1); + } +#if defined(_MSC_VER) + struct __stat64 statbuf; + r = _fstat64(fd, &statbuf); + if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ +#elif defined(__MINGW32__) && defined (__MSVCRT__) + struct _stati64 statbuf; + r = _fstati64(fd, &statbuf); + if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ +#else + struct stat statbuf; + r = fstat(fd, &statbuf); + if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ +#endif + return (U64)statbuf.st_size; +} + + UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) { int r; -- cgit v0.12 From 86edeb6784c25d5a952a9e5d33b25846f444375c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Tue, 25 Aug 2020 03:05:01 +0200 Subject: fixup: C90 compliance, clean up stat() types/calls --- programs/util.h | 58 ++++++++++++++++++++++++++++++--------------------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/programs/util.h b/programs/util.h index 4b2c09a..6658bf9 100644 --- a/programs/util.h +++ b/programs/util.h @@ -122,6 +122,27 @@ extern "C" { #endif +/*-**************************************** +* stat() functions +******************************************/ +#if defined(_MSC_VER) +# define UTIL_TYPE_stat __stat64 +# define UTIL_stat _stat64 +# define UTIL_fstat _fstat64 +# define UTIL_STAT_MODE_ISREG(st_mode) ((st_mode) & S_IFREG) +#elif defined(__MINGW32__) && defined (__MSVCRT__) +# define UTIL_TYPE_stat _stati64 +# define UTIL_stat _stati64 +# define UTIL_fstat _fstati64 +# define UTIL_STAT_MODE_ISREG(st_mode) ((st_mode) & S_IFREG) +#else +# define UTIL_TYPE_stat stat +# define UTIL_stat stat +# define UTIL_fstat fstat +# define UTIL_STAT_MODE_ISREG(st_mode) (S_ISREG(st_mode)) +#endif + + /* ************************************* * Constants ***************************************/ @@ -360,24 +381,16 @@ UTIL_STATIC U32 UTIL_isDirectory(const char* infilename) UTIL_STATIC U64 UTIL_getOpenFileSize(FILE* file) { int r; - int fd = fileno(file); + int fd; + struct UTIL_TYPE_stat statbuf; + + fd = fileno(file); if (fd < 0) { perror("fileno"); exit(1); } -#if defined(_MSC_VER) - struct __stat64 statbuf; - r = _fstat64(fd, &statbuf); - if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ -#elif defined(__MINGW32__) && defined (__MSVCRT__) - struct _stati64 statbuf; - r = _fstati64(fd, &statbuf); - if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ -#else - struct stat statbuf; - r = fstat(fd, &statbuf); - if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ -#endif + r = UTIL_fstat(fd, &statbuf); + if (r || !UTIL_STAT_MODE_ISREG(statbuf.st_mode)) return 0; /* No good... */ return (U64)statbuf.st_size; } @@ -385,19 +398,10 @@ UTIL_STATIC U64 UTIL_getOpenFileSize(FILE* file) UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) { int r; -#if defined(_MSC_VER) - struct __stat64 statbuf; - r = _stat64(infilename, &statbuf); - if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ -#elif defined(__MINGW32__) && defined (__MSVCRT__) - struct _stati64 statbuf; - r = _stati64(infilename, &statbuf); - if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ -#else - struct stat statbuf; - r = stat(infilename, &statbuf); - if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ -#endif + struct UTIL_TYPE_stat statbuf; + + r = UTIL_stat(infilename, &statbuf); + if (r || !UTIL_STAT_MODE_ISREG(statbuf.st_mode)) return 0; /* No good... */ return (U64)statbuf.st_size; } -- cgit v0.12 From a58b7ac6ce71819b712d0663f1d37bfb3cdd161c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Tue, 25 Aug 2020 03:37:41 +0200 Subject: fixup: Portable fileno() --- programs/util.h | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/programs/util.h b/programs/util.h index 6658bf9..2f3d0a8 100644 --- a/programs/util.h +++ b/programs/util.h @@ -143,6 +143,15 @@ extern "C" { #endif +/*-**************************************** +* fileno() function +******************************************/ +#if defined(_MSC_VER) +# define UTIL_fileno _fileno +#else +# define UTIL_fileno fileno +#endif + /* ************************************* * Constants ***************************************/ @@ -384,7 +393,7 @@ UTIL_STATIC U64 UTIL_getOpenFileSize(FILE* file) int fd; struct UTIL_TYPE_stat statbuf; - fd = fileno(file); + fd = UTIL_fileno(file); if (fd < 0) { perror("fileno"); exit(1); -- cgit v0.12 From 65f081ccaffce6753ec02c53c7b6413deddf2eb9 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 25 Aug 2020 14:25:51 -0700 Subject: use variables for program invocation --- tests/Makefile | 162 +++++++++++++++++++++++++++++---------------------------- 1 file changed, 82 insertions(+), 80 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index 05cae15..6f29020 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -146,7 +146,9 @@ ifneq (,$(filter $(shell uname),SunOS)) DIFF:=gdiff endif +CAT:=cat DD:=dd +DATAGEN:=./datagen .PHONY: list list: @@ -168,14 +170,14 @@ lz4_all.o: lz4_all.c $(CC) $(CFLAGS) $(CPPFLAGS) -c $^ -o $@ lz4_all.c: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c - cat $^ > $@ + $(CAT) $^ > $@ test-install: lz4 lib liblz4.pc lz4_root=.. ./test_install.sh test-lz4-sparse: lz4 datagen @echo "\n ---- test sparse file support ----" - ./datagen -g5M -P100 > tmplsdg5M + $(DATAGEN) -g5M -P100 > tmplsdg5M $(LZ4) -B4D tmplsdg5M -c | $(LZ4) -dv --sparse > tmplscB4 $(DIFF) -s tmplsdg5M tmplscB4 $(LZ4) -B5D tmplsdg5M -c | $(LZ4) -dv --sparse > tmplscB5 @@ -187,17 +189,17 @@ test-lz4-sparse: lz4 datagen $(LZ4) tmplsdg5M -c | $(LZ4) -dv --no-sparse > tmplsnosparse $(DIFF) -s tmplsdg5M tmplsnosparse ls -ls tmpls* - ./datagen -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > tmplsodd # Odd size file (to generate non-full last block) - ./datagen -s1 -g1200007 -P100 | $(DIFF) -s - tmplsodd + $(DATAGEN) -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > tmplsodd # Odd size file (to generate non-full last block) + $(DATAGEN) -s1 -g1200007 -P100 | $(DIFF) -s - tmplsodd ls -ls tmplsodd @$(RM) tmpls* @echo "\n Compatibility with Console :" echo "Hello World 1 !" | $(LZ4) | $(LZ4) -d -c - echo "Hello World 2 !" | $(LZ4) | $(LZ4) -d | cat + echo "Hello World 2 !" | $(LZ4) | $(LZ4) -d | $(CAT) echo "Hello World 3 !" | $(LZ4) --no-frame-crc | $(LZ4) -d -c @echo "\n Compatibility with Append :" - ./datagen -P100 -g1M > tmplsdg1M - cat tmplsdg1M tmplsdg1M > tmpls2M + $(DATAGEN) -P100 -g1M > tmplsdg1M + $(CAT) tmplsdg1M tmplsdg1M > tmpls2M $(LZ4) -B5 -v tmplsdg1M tmplsc $(LZ4) -d -v tmplsc tmplsr $(LZ4) -d -v tmplsc -c >> tmplsr @@ -207,7 +209,7 @@ test-lz4-sparse: lz4 datagen test-lz4-contentSize: lz4 datagen @echo "\n ---- test original size support ----" - ./datagen -g15M > tmplc1 + $(DATAGEN) -g15M > tmplc1 $(LZ4) -v tmplc1 -c | $(LZ4) -t $(LZ4) -v --content-size tmplc1 -c | $(LZ4) -d > tmplc2 $(DIFF) -s tmplc1 tmplc2 @@ -217,10 +219,10 @@ test-lz4-frame-concatenation: lz4 datagen @echo "\n ---- test frame concatenation ----" @echo -n > tmp-lfc-empty @echo hi > tmp-lfc-nonempty - cat tmp-lfc-nonempty tmp-lfc-empty tmp-lfc-nonempty > tmp-lfc-src + $(CAT) tmp-lfc-nonempty tmp-lfc-empty tmp-lfc-nonempty > tmp-lfc-src $(LZ4) -zq tmp-lfc-empty -c > tmp-lfc-empty.lz4 $(LZ4) -zq tmp-lfc-nonempty -c > tmp-lfc-nonempty.lz4 - cat tmp-lfc-nonempty.lz4 tmp-lfc-empty.lz4 tmp-lfc-nonempty.lz4 > tmp-lfc-concat.lz4 + $(CAT) tmp-lfc-nonempty.lz4 tmp-lfc-empty.lz4 tmp-lfc-nonempty.lz4 > tmp-lfc-concat.lz4 $(LZ4) -d tmp-lfc-concat.lz4 -c > tmp-lfc-result $(CMP) tmp-lfc-src tmp-lfc-result @$(RM) tmp-lfc-* @@ -228,9 +230,9 @@ test-lz4-frame-concatenation: lz4 datagen test-lz4-multiple: lz4 datagen @echo "\n ---- test multiple files ----" - @./datagen -s1 > tmp-tlm1 2> $(VOID) - @./datagen -s2 -g100K > tmp-tlm2 2> $(VOID) - @./datagen -s3 -g200K > tmp-tlm3 2> $(VOID) + @$(DATAGEN) -s1 > tmp-tlm1 2> $(VOID) + @$(DATAGEN) -s2 -g100K > tmp-tlm2 2> $(VOID) + @$(DATAGEN) -s3 -g200K > tmp-tlm3 2> $(VOID) # compress multiple files : one .lz4 per source file $(LZ4) -f -m tmp-tlm* test -f tmp-tlm1.lz4 @@ -245,7 +247,7 @@ test-lz4-multiple: lz4 datagen $(CMP) tmp-tlm2 tmp-tlm2-orig $(CMP) tmp-tlm3 tmp-tlm3-orig # compress multiple files into stdout - cat tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 > tmp-tlm-concat1 + $(CAT) tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 > tmp-tlm-concat1 $(RM) *.lz4 $(LZ4) -m tmp-tlm1 tmp-tlm2 tmp-tlm3 -c > tmp-tlm-concat2 test ! -f tmp-tlm1.lz4 # must not create .lz4 artefact @@ -253,7 +255,7 @@ test-lz4-multiple: lz4 datagen # decompress multiple files into stdout $(RM) tmp-tlm-concat1 tmp-tlm-concat2 $(LZ4) -f -m tmp-tlm1 tmp-tlm2 tmp-tlm3 # generate .lz4 to decompress - cat tmp-tlm1 tmp-tlm2 tmp-tlm3 > tmp-tlm-concat1 # create concatenated reference + $(CAT) tmp-tlm1 tmp-tlm2 tmp-tlm3 > tmp-tlm-concat1 # create concatenated reference $(RM) tmp-tlm1 tmp-tlm2 tmp-tlm3 $(LZ4) -d -m tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 -c > tmp-tlm-concat2 test ! -f tmp-tlm1 # must not create file artefact @@ -264,9 +266,9 @@ test-lz4-multiple: lz4 datagen test-lz4-multiple-legacy: lz4 datagen @echo "\n ---- test multiple files (Legacy format) ----" - @./datagen -s1 > tmp-tlm1 2> $(VOID) - @./datagen -s2 -g100K > tmp-tlm2 2> $(VOID) - @./datagen -s3 -g200K > tmp-tlm3 2> $(VOID) + @$(DATAGEN) -s1 > tmp-tlm1 2> $(VOID) + @$(DATAGEN) -s2 -g100K > tmp-tlm2 2> $(VOID) + @$(DATAGEN) -s3 -g200K > tmp-tlm3 2> $(VOID) # compress multiple files using legacy format: one .lz4 per source file $(LZ4) -f -l -m tmp-tlm* test -f tmp-tlm1.lz4 @@ -282,7 +284,7 @@ test-lz4-multiple-legacy: lz4 datagen $(CMP) tmp-tlm2 tmp-tlm2-orig $(CMP) tmp-tlm3 tmp-tlm3-orig # compress multiple files into stdout using legacy format - cat tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 > tmp-tlm-concat1 + $(CAT) tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 > tmp-tlm-concat1 $(RM) *.lz4 $(LZ4) -l -m tmp-tlm1 tmp-tlm2 tmp-tlm3 -c > tmp-tlm-concat2 test ! -f tmp-tlm1.lz4 # must not create .lz4 artefact @@ -290,7 +292,7 @@ test-lz4-multiple-legacy: lz4 datagen # # # decompress multiple files into stdout using legacy format $(RM) tmp-tlm-concat1 tmp-tlm-concat2 $(LZ4) -l -f -m tmp-tlm1 tmp-tlm2 tmp-tlm3 # generate .lz4 to decompress - cat tmp-tlm1 tmp-tlm2 tmp-tlm3 > tmp-tlm-concat1 # create concatenated reference + $(CAT) tmp-tlm1 tmp-tlm2 tmp-tlm3 > tmp-tlm-concat1 # create concatenated reference $(RM) tmp-tlm1 tmp-tlm2 tmp-tlm3 $(LZ4) -d -m tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 -c > tmp-tlm-concat2 $(LZ4) -d -l -m tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 -c > tmp-tlm-concat2 # -l mustn't impact option -d @@ -302,18 +304,18 @@ test-lz4-multiple-legacy: lz4 datagen test-lz4-basic: lz4 datagen unlz4 lz4cat @echo "\n ---- test lz4 basic compression/decompression ----" - ./datagen -g0 | $(LZ4) -v | $(LZ4) -t - ./datagen -g16KB | $(LZ4) -9 | $(LZ4) -t - ./datagen -g20KB > tmp-tlb-dg20k + $(DATAGEN) -g0 | $(LZ4) -v | $(LZ4) -t + $(DATAGEN) -g16KB | $(LZ4) -9 | $(LZ4) -t + $(DATAGEN) -g20KB > tmp-tlb-dg20k $(LZ4) < tmp-tlb-dg20k | $(LZ4) -d > tmp-tlb-dec $(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec $(LZ4) --no-frame-crc < tmp-tlb-dg20k | $(LZ4) -d > tmp-tlb-dec $(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec - ./datagen | $(LZ4) -BI | $(LZ4) -t - ./datagen -g6M -P99 | $(LZ4) -9BD | $(LZ4) -t - ./datagen -g17M | $(LZ4) -9v | $(LZ4) -qt - ./datagen -g33M | $(LZ4) --no-frame-crc | $(LZ4) -t - ./datagen -g256MB | $(LZ4) -vqB4D | $(LZ4) -t + $(DATAGEN) | $(LZ4) -BI | $(LZ4) -t + $(DATAGEN) -g6M -P99 | $(LZ4) -9BD | $(LZ4) -t + $(DATAGEN) -g17M | $(LZ4) -9v | $(LZ4) -qt + $(DATAGEN) -g33M | $(LZ4) --no-frame-crc | $(LZ4) -t + $(DATAGEN) -g256MB | $(LZ4) -vqB4D | $(LZ4) -t @echo "hello world" > tmp-tlb-hw $(LZ4) --rm -f tmp-tlb-hw tmp-tlb-hw.lz4 test ! -f tmp-tlb-hw # must fail (--rm) @@ -342,18 +344,18 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat $(DIFF) -q tmp-tlb-hw tmp-tlb4 $(LZ4) -f tmp-tlb-hw $(LZ4) --list tmp-tlb-hw.lz4 # test --list on valid single-frame file - cat tmp-tlb-hw >> tmp-tlb-hw.lz4 + $(CAT) tmp-tlb-hw >> tmp-tlb-hw.lz4 $(LZ4) -f tmp-tlb-hw.lz4 # uncompress valid frame followed by invalid data $(LZ4) -BX tmp-tlb-hw -c -q | $(LZ4) -tv # test block checksum - # ./datagen -g20KB generates the same file every single time - # cannot save output of ./datagen -g20KB as input file to lz4 because the following shell commands are run before ./datagen -g20KB - test "$(shell ./datagen -g20KB | $(LZ4) -c --fast | wc -c)" -lt "$(shell ./datagen -g20KB | $(LZ4) -c --fast=9 | wc -c)" # -1 vs -9 - test "$(shell ./datagen -g20KB | $(LZ4) -c -1 | wc -c)" -lt "$(shell ./datagen -g20KB| $(LZ4) -c --fast=1 | wc -c)" # 1 vs -1 - test "$(shell ./datagen -g20KB | $(LZ4) -c --fast=1 | wc -c)" -eq "$(shell ./datagen -g20KB| $(LZ4) -c --fast| wc -c)" # checks default fast compression is -1 + # $(DATAGEN) -g20KB generates the same file every single time + # cannot save output of $(DATAGEN) -g20KB as input file to lz4 because the following shell commands are run before $(DATAGEN) -g20KB + test "$(shell $(DATAGEN) -g20KB | $(LZ4) -c --fast | wc -c)" -lt "$(shell $(DATAGEN) -g20KB | $(LZ4) -c --fast=9 | wc -c)" # -1 vs -9 + test "$(shell $(DATAGEN) -g20KB | $(LZ4) -c -1 | wc -c)" -lt "$(shell $(DATAGEN) -g20KB| $(LZ4) -c --fast=1 | wc -c)" # 1 vs -1 + test "$(shell $(DATAGEN) -g20KB | $(LZ4) -c --fast=1 | wc -c)" -eq "$(shell $(DATAGEN) -g20KB| $(LZ4) -c --fast| wc -c)" # checks default fast compression is -1 ! $(LZ4) -c --fast=0 tmp-tlb-dg20K # lz4 should fail when fast=0 ! $(LZ4) -c --fast=-1 tmp-tlb-dg20K # lz4 should fail when fast=-1 # High --fast values can result in out-of-bound dereferences #876 - ./datagen -g1M | $(LZ4) -c --fast=999999999 > /dev/null + $(DATAGEN) -g1M | $(LZ4) -c --fast=999999999 > /dev/null # Test for #596 @echo "TEST" > tmp-tlb-test $(LZ4) -m tmp-tlb-test @@ -365,10 +367,10 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat test-lz4-dict: lz4 datagen @echo "\n ---- test lz4 compression/decompression with dictionary ----" - ./datagen -g16KB > tmp-dict - ./datagen -g32KB > tmp-dict-sample-32k + $(DATAGEN) -g16KB > tmp-dict + $(DATAGEN) -g32KB > tmp-dict-sample-32k < tmp-dict-sample-32k $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-32k - ./datagen -g128MB > tmp-dict-sample-128m + $(DATAGEN) -g128MB > tmp-dict-sample-128m < tmp-dict-sample-128m $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-128m touch tmp-dict-sample-0 < tmp-dict-sample-0 $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-0 @@ -377,10 +379,10 @@ test-lz4-dict: lz4 datagen < tmp-dict-sample-0 $(LZ4) -D tmp-dict-sample-0 | $(LZ4) -dD tmp-dict-sample-0 | diff - tmp-dict-sample-0 @echo "\n ---- test lz4 dictionary loading ----" - ./datagen -g128KB > tmp-dict-data-128KB + $(DATAGEN) -g128KB > tmp-dict-data-128KB set -e; \ for l in 0 1 4 128 32767 32768 32769 65535 65536 65537 98303 98304 98305 131071 131072 131073; do \ - ./datagen -g$$l > tmp-dict-$$l; \ + $(DATAGEN) -g$$l > tmp-dict-$$l; \ $(DD) if=tmp-dict-$$l of=tmp-dict-$$l-tail bs=1 count=65536 skip=$$((l > 65536 ? l - 65536 : 0)); \ < tmp-dict-$$l $(LZ4) -D stdin tmp-dict-data-128KB -c | $(LZ4) -dD tmp-dict-$$l-tail | $(DIFF) - tmp-dict-data-128KB; \ < tmp-dict-$$l-tail $(LZ4) -D stdin tmp-dict-data-128KB -c | $(LZ4) -dD tmp-dict-$$l | $(DIFF) - tmp-dict-data-128KB; \ @@ -390,12 +392,12 @@ test-lz4-dict: lz4 datagen test-lz4-hugefile: lz4 datagen @echo "\n ---- test huge files compression/decompression ----" - ./datagen -g6GB | $(LZ4) -vB5D | $(LZ4) -qt - ./datagen -g5GB | $(LZ4) -v4BD | $(LZ4) -qt + $(DATAGEN) -g6GB | $(LZ4) -vB5D | $(LZ4) -qt + $(DATAGEN) -g5GB | $(LZ4) -v4BD | $(LZ4) -qt # test large file size [2-4] GB - @./datagen -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmphf1 + @$(DATAGEN) -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmphf1 @ls -ls tmphf1 - @./datagen -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmphf2 + @$(DATAGEN) -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmphf2 @ls -ls tmphf2 $(DIFF) -s tmphf1 tmphf2 @$(RM) tmphf* @@ -404,14 +406,14 @@ test-lz4-testmode: lz4 datagen @echo "\n ---- bench mode ----" $(LZ4) -bi1 @echo "\n ---- test mode ----" - ! ./datagen | $(LZ4) -t - ! ./datagen | $(LZ4) -tf + ! $(DATAGEN) | $(LZ4) -t + ! $(DATAGEN) | $(LZ4) -tf @echo "\n ---- pass-through mode ----" @echo "Why hello there " > tmp-tlt2.lz4 ! $(LZ4) -f tmp-tlt2.lz4 > $(VOID) - ! ./datagen | $(LZ4) -dc > $(VOID) - ! ./datagen | $(LZ4) -df > $(VOID) - ./datagen | $(LZ4) -dcf > $(VOID) + ! $(DATAGEN) | $(LZ4) -dc > $(VOID) + ! $(DATAGEN) | $(LZ4) -df > $(VOID) + $(DATAGEN) | $(LZ4) -dcf > $(VOID) @echo "Hello World !" > tmp-tlt1 $(LZ4) -dcf tmp-tlt1 @echo "from underground..." > tmp-tlt2 @@ -425,16 +427,16 @@ test-lz4-testmode: lz4 datagen test-lz4-opt-parser: lz4 datagen @echo "\n ---- test opt-parser ----" - ./datagen -g16KB | $(LZ4) -12 | $(LZ4) -t - ./datagen -P10 | $(LZ4) -12B4 | $(LZ4) -t - ./datagen -g256K | $(LZ4) -12B4D | $(LZ4) -t - ./datagen -g512K -P25 | $(LZ4) -12BD | $(LZ4) -t - ./datagen -g1M | $(LZ4) -12B5 | $(LZ4) -t - ./datagen -g2M -P99 | $(LZ4) -11B4D | $(LZ4) -t - ./datagen -g4M | $(LZ4) -11vq | $(LZ4) -qt - ./datagen -g8M | $(LZ4) -11B4 | $(LZ4) -t - ./datagen -g16M -P90 | $(LZ4) -11B5 | $(LZ4) -t - ./datagen -g32M -P10 | $(LZ4) -11B5D | $(LZ4) -t + $(DATAGEN) -g16KB | $(LZ4) -12 | $(LZ4) -t + $(DATAGEN) -P10 | $(LZ4) -12B4 | $(LZ4) -t + $(DATAGEN) -g256K | $(LZ4) -12B4D | $(LZ4) -t + $(DATAGEN) -g512K -P25 | $(LZ4) -12BD | $(LZ4) -t + $(DATAGEN) -g1M | $(LZ4) -12B5 | $(LZ4) -t + $(DATAGEN) -g2M -P99 | $(LZ4) -11B4D | $(LZ4) -t + $(DATAGEN) -g4M | $(LZ4) -11vq | $(LZ4) -qt + $(DATAGEN) -g8M | $(LZ4) -11B4 | $(LZ4) -t + $(DATAGEN) -g16M -P90 | $(LZ4) -11B5 | $(LZ4) -t + $(DATAGEN) -g32M -P10 | $(LZ4) -11B5D | $(LZ4) -t test-lz4-essentials : lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-multiple-legacy \ test-lz4-frame-concatenation test-lz4-testmode \ @@ -447,35 +449,35 @@ test-lz4: lz4 datagen test-lz4-essentials test-lz4-opt-parser \ test-lz4c: lz4c datagen @echo "\n ---- test lz4c variant ----" - ./datagen -g256MB | $(LZ4)c -l -v | $(LZ4)c -t + $(DATAGEN) -g256MB | $(LZ4)c -l -v | $(LZ4)c -t test-lz4c32: CFLAGS+=-m32 test-lz4c32: test-lz4 test-interop-32-64: lz4 lz4c32 datagen @echo "\n ---- test interoperability 32-bits -vs- 64 bits ----" - ./datagen -g16KB | $(LZ4)c32 -9 | $(LZ4) -t - ./datagen -P10 | $(LZ4) -9B4 | $(LZ4)c32 -t - ./datagen | $(LZ4)c32 | $(LZ4) -t - ./datagen -g1M | $(LZ4) -3B5 | $(LZ4)c32 -t - ./datagen -g256MB | $(LZ4)c32 -vqB4D | $(LZ4) -qt - ./datagen -g1G -P90 | $(LZ4) | $(LZ4)c32 -t - ./datagen -g6GB | $(LZ4)c32 -vq9BD | $(LZ4) -qt + $(DATAGEN) -g16KB | $(LZ4)c32 -9 | $(LZ4) -t + $(DATAGEN) -P10 | $(LZ4) -9B4 | $(LZ4)c32 -t + $(DATAGEN) | $(LZ4)c32 | $(LZ4) -t + $(DATAGEN) -g1M | $(LZ4) -3B5 | $(LZ4)c32 -t + $(DATAGEN) -g256MB | $(LZ4)c32 -vqB4D | $(LZ4) -qt + $(DATAGEN) -g1G -P90 | $(LZ4) | $(LZ4)c32 -t + $(DATAGEN) -g6GB | $(LZ4)c32 -vq9BD | $(LZ4) -qt test-lz4c32-basic: lz4c32 datagen @echo "\n ---- test lz4c32 32-bits version ----" - ./datagen -g16KB | $(LZ4)c32 -9 | $(LZ4)c32 -t - ./datagen | $(LZ4)c32 | $(LZ4)c32 -t - ./datagen -g256MB | $(LZ4)c32 -vqB4D | $(LZ4)c32 -qt - ./datagen -g6GB | $(LZ4)c32 -vqB5D | $(LZ4)c32 -qt + $(DATAGEN) -g16KB | $(LZ4)c32 -9 | $(LZ4)c32 -t + $(DATAGEN) | $(LZ4)c32 | $(LZ4)c32 -t + $(DATAGEN) -g256MB | $(LZ4)c32 -vqB4D | $(LZ4)c32 -qt + $(DATAGEN) -g6GB | $(LZ4)c32 -vqB5D | $(LZ4)c32 -qt test-platform: @echo "\n ---- test lz4 $(QEMU_SYS) platform ----" - $(QEMU_SYS) ./datagen -g16KB | $(QEMU_SYS) $(LZ4) -9 | $(QEMU_SYS) $(LZ4) -t - $(QEMU_SYS) ./datagen | $(QEMU_SYS) $(LZ4) | $(QEMU_SYS) $(LZ4) -t - $(QEMU_SYS) ./datagen -g256MB | $(QEMU_SYS) $(LZ4) -vqB4D | $(QEMU_SYS) $(LZ4) -qt + $(QEMU_SYS) $(DATAGEN) -g16KB | $(QEMU_SYS) $(LZ4) -9 | $(QEMU_SYS) $(LZ4) -t + $(QEMU_SYS) $(DATAGEN) | $(QEMU_SYS) $(LZ4) | $(QEMU_SYS) $(LZ4) -t + $(QEMU_SYS) $(DATAGEN) -g256MB | $(QEMU_SYS) $(LZ4) -vqB4D | $(QEMU_SYS) $(LZ4) -qt ifneq ($(QEMU_SYS),qemu-arm-static) - $(QEMU_SYS) ./datagen -g3GB | $(QEMU_SYS) $(LZ4) -vqB5D | $(QEMU_SYS) $(LZ4) -qt + $(QEMU_SYS) $(DATAGEN) -g3GB | $(QEMU_SYS) $(LZ4) -vqB5D | $(QEMU_SYS) $(LZ4) -qt endif test-fullbench: fullbench @@ -498,13 +500,13 @@ test-frametest32: test-frametest test-mem: lz4 datagen fuzzer frametest fullbench @echo "\n ---- valgrind tests : memory analyzer ----" - valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID) - ./datagen -g16KB > ftmdg16K + valgrind --leak-check=yes --error-exitcode=1 $(DATAGEN) -g50M > $(VOID) + $(DATAGEN) -g16KB > ftmdg16K valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -BD -f ftmdg16K $(VOID) - ./datagen -g16KB -s2 > ftmdg16K2 - ./datagen -g16KB -s3 > ftmdg16K3 + $(DATAGEN) -g16KB -s2 > ftmdg16K2 + $(DATAGEN) -g16KB -s3 > ftmdg16K3 valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --force --multiple ftmdg16K ftmdg16K2 ftmdg16K3 - ./datagen -g7MB > ftmdg7M + $(DATAGEN) -g7MB > ftmdg7M valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f ftmdg7M ftmdg16K2 valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -t ftmdg16K2 valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 ftmdg7M -- cgit v0.12 From 224735ea1b7b224e14caf2e39b21854c439a7179 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 25 Aug 2020 14:42:15 -0700 Subject: added more tests around --content-size notably in association with `stdin` --- tests/Makefile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index 6f29020..2fe9e69 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -212,7 +212,16 @@ test-lz4-contentSize: lz4 datagen $(DATAGEN) -g15M > tmplc1 $(LZ4) -v tmplc1 -c | $(LZ4) -t $(LZ4) -v --content-size tmplc1 -c | $(LZ4) -d > tmplc2 - $(DIFF) -s tmplc1 tmplc2 + $(DIFF) tmplc1 tmplc2 + $(LZ4) -f tmplc1 + $(LZ4) --content-size tmplc1 -c > tmplc2.lz4 + ! $(DIFF) tmplc1.lz4 tmplc2.lz4 # must differ, due to content size + $(LZ4) --content-size < tmplc1 > tmplc3.lz4 + $(DIFF) tmplc2.lz4 tmplc3.lz4 # both must contain content size + $(CAT) tmplc1 | $(LZ4) > tmplc4.lz4 + $(DIFF) tmplc1.lz4 tmplc4.lz4 # both don't have content size + $(CAT) tmplc1 | $(LZ4) --content-size > tmplc5.lz4 # can't determine content size + $(DIFF) tmplc1.lz4 tmplc5.lz4 # both don't have content size @$(RM) tmplc* test-lz4-frame-concatenation: lz4 datagen -- cgit v0.12 From 5b3f9453108f94aef856304db8146cde54bdcbb8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 25 Aug 2020 14:53:08 -0700 Subject: fix test under non-interactive environment --- tests/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index 2fe9e69..866ff5d 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -213,7 +213,7 @@ test-lz4-contentSize: lz4 datagen $(LZ4) -v tmplc1 -c | $(LZ4) -t $(LZ4) -v --content-size tmplc1 -c | $(LZ4) -d > tmplc2 $(DIFF) tmplc1 tmplc2 - $(LZ4) -f tmplc1 + $(LZ4) -f tmplc1 -c > tmplc1.lz4 $(LZ4) --content-size tmplc1 -c > tmplc2.lz4 ! $(DIFF) tmplc1.lz4 tmplc2.lz4 # must differ, due to content size $(LZ4) --content-size < tmplc1 > tmplc3.lz4 -- cgit v0.12