summaryrefslogtreecommitdiffstats
path: root/tests
diff options
context:
space:
mode:
Diffstat (limited to 'tests')
-rw-r--r--tests/.gitignore4
-rw-r--r--tests/Makefile490
-rw-r--r--tests/README.md6
-rw-r--r--tests/abiTest.c218
-rw-r--r--tests/checkFrame.c4
-rw-r--r--tests/checkTag.c2
-rwxr-xr-xtests/check_liblz4_version.sh6
-rw-r--r--tests/datagencli.c43
-rw-r--r--tests/decompress-partial-usingDict.c103
-rw-r--r--tests/frametest.c140
-rw-r--r--tests/freestanding.c239
-rw-r--r--tests/fullbench.c44
-rw-r--r--tests/fuzzer.c75
-rw-r--r--tests/goldenSamples/skip.binbin0 -> 38 bytes
-rw-r--r--tests/roundTripTest.c10
-rw-r--r--tests/test-lz4-abi.py173
-rw-r--r--tests/test-lz4-list.py93
-rw-r--r--tests/test-lz4-speed.py6
-rwxr-xr-xtests/test_custom_block_sizes.sh4
-rwxr-xr-xtests/test_install.sh4
-rw-r--r--tests/unicode_lint.sh48
21 files changed, 1360 insertions, 352 deletions
diff --git a/tests/.gitignore b/tests/.gitignore
index 99351af..c7d8f19 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -12,10 +12,14 @@ roundTripTest
checkTag
checkFrame
decompress-partial
+decompress-partial-usingDict
+abiTest
+freestanding
# test artefacts
tmp*
versionsTest
+abiTests
lz4_all.c
# local tests
diff --git a/tests/Makefile b/tests/Makefile
index 6eee132..93a5581 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -1,6 +1,6 @@
# ##########################################################################
# LZ4 programs - Makefile
-# Copyright (C) Yann Collet 2011-present
+# Copyright (C) Yann Collet 2011-2020
#
# GPL v2 License
#
@@ -112,6 +112,12 @@ checkFrame : lz4frame.o lz4.o lz4hc.o xxhash.o checkFrame.c
decompress-partial: lz4.o decompress-partial.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
+decompress-partial-usingDict: lz4.o decompress-partial-usingDict.c
+ $(CC) $(FLAGS) $^ -o $@$(EXT)
+
+freestanding: freestanding.c
+ $(CC) -ffreestanding -nostdlib $^ -o $@$(EXT)
+
.PHONY: clean
clean:
@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
@@ -124,7 +130,8 @@ clean:
fasttest$(EXT) roundTripTest$(EXT) \
datagen$(EXT) checkTag$(EXT) \
frameTest$(EXT) decompress-partial$(EXT) \
- lz4_all.c
+ abiTest$(EXT) freestanding$(EXT) \
+ lz4_all.c
@$(RM) -rf $(TESTDIR)
@echo Cleaning completed
@@ -136,6 +143,12 @@ versionsTest:
listTest: lz4
QEMU_SYS=$(QEMU_SYS) $(PYTHON) test-lz4-list.py
+abiTest: LDLIBS += -llz4
+
+.PHONY: abiTests
+abiTests:
+ $(PYTHON) test-lz4-abi.py
+
checkTag: checkTag.c $(LZ4DIR)/lz4.h
$(CC) $(FLAGS) $< -o $@$(EXT)
@@ -145,14 +158,15 @@ checkTag: checkTag.c $(LZ4DIR)/lz4.h
ifeq ($(POSIX_ENV),Yes)
MD5:=md5sum
-ifneq (,$(filter $(shell uname), Darwin ))
+ifneq (,$(filter $(shell $(UNAME)), Darwin ))
MD5:=md5 -r
endif
# note : we should probably settle on a single compare utility
CMP:=cmp
+GREP:=grep
DIFF:=diff
-ifneq (,$(filter $(shell uname),SunOS))
+ifneq (,$(filter $(shell $(UNAME)),SunOS))
DIFF:=gdiff
endif
@@ -168,278 +182,327 @@ list:
check: test-lz4-essentials
.PHONY: test
-test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-install test-amalgamation listTest test-decompress-partial
+test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-amalgamation listTest test-decompress-partial
.PHONY: test32
test32: CFLAGS+=-m32
test32: test
+.PHONY: test-amalgamation
test-amalgamation: lz4_all.o
lz4_all.c: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c
$(CAT) $^ > $@
+.PHONY: test-install
test-install: lz4 lib liblz4.pc
lz4_root=.. ./test_install.sh
+.PHONY: test-compile-with-lz4-memory-usage
+test-compile-with-lz4-memory-usage:
+ $(MAKE) clean; CFLAGS=-O0 CPPFLAGS=-D'LZ4_MEMORY_USAGE=LZ4_MEMORY_USAGE_MIN' $(MAKE) all
+ $(MAKE) clean; CFLAGS=-O0 CPPFLAGS=-D'LZ4_MEMORY_USAGE=LZ4_MEMORY_USAGE_MAX' $(MAKE) all
+
+.PHONY: test-lz4-sparse
+# Rules regarding Temporary test files :
+# Each test must use its own unique set of names during execution.
+# Each temporary test file must begin by an FPREFIX.
+# Each FPREFIX must be unique for each test.
+# All FPREFIX must start with `tmp`, for `make clean`
+# All tests must clean their temporary test files on successful completion,
+# and only their test files : do not employ sweeping statements such `rm tmp*` or `rm *.lz4`
+test-lz4-sparse: FPREFIX = tmp-tls
test-lz4-sparse: lz4 datagen
@echo "\n ---- test sparse file support ----"
- $(DATAGEN) -g5M -P100 > tmplsdg5M
- $(LZ4) -B4D tmplsdg5M -c | $(LZ4) -dv --sparse > tmplscB4
- $(DIFF) -s tmplsdg5M tmplscB4
- $(LZ4) -B5D tmplsdg5M -c | $(LZ4) -dv --sparse > tmplscB5
- $(DIFF) -s tmplsdg5M tmplscB5
- $(LZ4) -B6D tmplsdg5M -c | $(LZ4) -dv --sparse > tmplscB6
- $(DIFF) -s tmplsdg5M tmplscB6
- $(LZ4) -B7D tmplsdg5M -c | $(LZ4) -dv --sparse > tmplscB7
- $(DIFF) -s tmplsdg5M tmplscB7
- $(LZ4) tmplsdg5M -c | $(LZ4) -dv --no-sparse > tmplsnosparse
- $(DIFF) -s tmplsdg5M tmplsnosparse
- ls -ls tmpls*
- $(DATAGEN) -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > tmplsodd # Odd size file (to generate non-full last block)
- $(DATAGEN) -s1 -g1200007 -P100 | $(DIFF) -s - tmplsodd
- ls -ls tmplsodd
- @$(RM) tmpls*
+ $(DATAGEN) -g5M -P100 > $(FPREFIX)dg5M
+ $(LZ4) -B4D $(FPREFIX)dg5M -c | $(LZ4) -dv --sparse > $(FPREFIX)cB4
+ $(DIFF) -s $(FPREFIX)dg5M $(FPREFIX)cB4
+ $(LZ4) -B5D $(FPREFIX)dg5M -c | $(LZ4) -dv --sparse > $(FPREFIX)cB5
+ $(DIFF) -s $(FPREFIX)dg5M $(FPREFIX)cB5
+ $(LZ4) -B6D $(FPREFIX)dg5M -c | $(LZ4) -dv --sparse > $(FPREFIX)cB6
+ $(DIFF) -s $(FPREFIX)dg5M $(FPREFIX)cB6
+ $(LZ4) -B7D $(FPREFIX)dg5M -c | $(LZ4) -dv --sparse > $(FPREFIX)cB7
+ $(DIFF) -s $(FPREFIX)dg5M $(FPREFIX)cB7
+ $(LZ4) $(FPREFIX)dg5M -c | $(LZ4) -dv --no-sparse > $(FPREFIX)nosparse
+ $(DIFF) -s $(FPREFIX)dg5M $(FPREFIX)nosparse
+ ls -ls $(FPREFIX)*
+ $(DATAGEN) -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > $(FPREFIX)odd # Odd size file (to generate non-full last block)
+ $(DATAGEN) -s1 -g1200007 -P100 | $(DIFF) -s - $(FPREFIX)odd
+ ls -ls $(FPREFIX)odd
+ @$(RM) $(FPREFIX)*
@echo "\n Compatibility with Console :"
echo "Hello World 1 !" | $(LZ4) | $(LZ4) -d -c
echo "Hello World 2 !" | $(LZ4) | $(LZ4) -d | $(CAT)
echo "Hello World 3 !" | $(LZ4) --no-frame-crc | $(LZ4) -d -c
@echo "\n Compatibility with Append :"
- $(DATAGEN) -P100 -g1M > tmplsdg1M
- $(CAT) tmplsdg1M tmplsdg1M > tmpls2M
- $(LZ4) -B5 -v tmplsdg1M tmplsc
- $(LZ4) -d -v tmplsc tmplsr
- $(LZ4) -d -v tmplsc -c >> tmplsr
- ls -ls tmp*
- $(DIFF) tmpls2M tmplsr
- @$(RM) tmpls*
-
+ $(DATAGEN) -P100 -g1M > $(FPREFIX)dg1M
+ $(CAT) $(FPREFIX)dg1M $(FPREFIX)dg1M > $(FPREFIX)2M
+ $(LZ4) -B5 -v $(FPREFIX)dg1M $(FPREFIX)c
+ $(LZ4) -d -v $(FPREFIX)c $(FPREFIX)r
+ $(LZ4) -d -v $(FPREFIX)c -c >> $(FPREFIX)r
+ ls -ls $(FPREFIX)*
+ $(DIFF) $(FPREFIX)2M $(FPREFIX)r
+ @$(RM) $(FPREFIX)*
+
+test-lz4-contentSize: FPREFIX = tmp-lzc
test-lz4-contentSize: lz4 datagen
@echo "\n ---- test original size support ----"
- $(DATAGEN) -g15M > tmplc1
- $(LZ4) -v tmplc1 -c | $(LZ4) -t
- $(LZ4) -v --content-size tmplc1 -c | $(LZ4) -d > tmplc2
- $(DIFF) tmplc1 tmplc2
- $(LZ4) -f tmplc1 -c > tmplc1.lz4
- $(LZ4) --content-size tmplc1 -c > tmplc2.lz4
- ! $(DIFF) tmplc1.lz4 tmplc2.lz4 # must differ, due to content size
- $(LZ4) --content-size < tmplc1 > tmplc3.lz4
- $(DIFF) tmplc2.lz4 tmplc3.lz4 # both must contain content size
- $(CAT) tmplc1 | $(LZ4) > tmplc4.lz4
- $(DIFF) tmplc1.lz4 tmplc4.lz4 # both don't have content size
- $(CAT) tmplc1 | $(LZ4) --content-size > tmplc5.lz4 # can't determine content size
- $(DIFF) tmplc1.lz4 tmplc5.lz4 # both don't have content size
- @$(RM) tmplc*
-
+ $(DATAGEN) -g15M > $(FPREFIX)
+ $(LZ4) -v $(FPREFIX) -c | $(LZ4) -t
+ $(LZ4) -v --content-size $(FPREFIX) -c | $(LZ4) -d > $(FPREFIX)-dup
+ $(DIFF) $(FPREFIX) $(FPREFIX)-dup
+ $(LZ4) -f $(FPREFIX) -c > $(FPREFIX).lz4 # compressed with content size
+ $(LZ4) --content-size $(FPREFIX) -c > $(FPREFIX)-wcz.lz4
+ ! $(DIFF) $(FPREFIX).lz4 $(FPREFIX)-wcz.lz4 # must differ, due to content size
+ $(LZ4) --content-size < $(FPREFIX) > $(FPREFIX)-wcz2.lz4 # can determine content size because stdin is just a file
+ $(DIFF) $(FPREFIX)-wcz.lz4 $(FPREFIX)-wcz2.lz4 # both must contain content size
+ $(CAT) $(FPREFIX) | $(LZ4) > $(FPREFIX)-ncz.lz4
+ $(DIFF) $(FPREFIX).lz4 $(FPREFIX)-ncz.lz4 # both don't have content size
+ $(CAT) $(FPREFIX) | $(LZ4) --content-size > $(FPREFIX)-ncz2.lz4 # can't determine content size
+ $(DIFF) $(FPREFIX).lz4 $(FPREFIX)-ncz2.lz4 # both don't have content size
+ @$(RM) $(FPREFIX)*
+
+test-lz4-frame-concatenation: FPREFIX = tmp-lfc
test-lz4-frame-concatenation: lz4 datagen
@echo "\n ---- test frame concatenation ----"
- @echo -n > tmp-lfc-empty
- @echo hi > tmp-lfc-nonempty
- $(CAT) tmp-lfc-nonempty tmp-lfc-empty tmp-lfc-nonempty > tmp-lfc-src
- $(LZ4) -zq tmp-lfc-empty -c > tmp-lfc-empty.lz4
- $(LZ4) -zq tmp-lfc-nonempty -c > tmp-lfc-nonempty.lz4
- $(CAT) tmp-lfc-nonempty.lz4 tmp-lfc-empty.lz4 tmp-lfc-nonempty.lz4 > tmp-lfc-concat.lz4
- $(LZ4) -d tmp-lfc-concat.lz4 -c > tmp-lfc-result
- $(CMP) tmp-lfc-src tmp-lfc-result
- @$(RM) tmp-lfc-*
+ @echo -n > $(FPREFIX)-empty
+ @echo hi > $(FPREFIX)-nonempty
+ $(CAT) $(FPREFIX)-nonempty $(FPREFIX)-empty $(FPREFIX)-nonempty > $(FPREFIX)-src
+ $(LZ4) -zq $(FPREFIX)-empty -c > $(FPREFIX)-empty.lz4
+ $(LZ4) -zq $(FPREFIX)-nonempty -c > $(FPREFIX)-nonempty.lz4
+ $(CAT) $(FPREFIX)-nonempty.lz4 $(FPREFIX)-empty.lz4 $(FPREFIX)-nonempty.lz4 > $(FPREFIX)-concat.lz4
+ $(LZ4) -d $(FPREFIX)-concat.lz4 -c > $(FPREFIX)-result
+ $(CMP) $(FPREFIX)-src $(FPREFIX)-result
+ @$(RM) $(FPREFIX)*
@echo frame concatenation test completed
+test-lz4-multiple: FPREFIX = tmp-tml
test-lz4-multiple: lz4 datagen
@echo "\n ---- test multiple files ----"
- @$(DATAGEN) -s1 > tmp-tlm1 2> $(VOID)
- @$(DATAGEN) -s2 -g100K > tmp-tlm2 2> $(VOID)
- @$(DATAGEN) -s3 -g200K > tmp-tlm3 2> $(VOID)
+ @$(DATAGEN) -s1 > $(FPREFIX)1 2> $(VOID)
+ @$(DATAGEN) -s2 -g100K > $(FPREFIX)2 2> $(VOID)
+ @$(DATAGEN) -s3 -g200K > $(FPREFIX)3 2> $(VOID)
# compress multiple files : one .lz4 per source file
- $(LZ4) -f -m tmp-tlm*
- test -f tmp-tlm1.lz4
- test -f tmp-tlm2.lz4
- test -f tmp-tlm3.lz4
+ $(LZ4) -f -m $(FPREFIX)*
+ test -f $(FPREFIX)1.lz4
+ test -f $(FPREFIX)2.lz4
+ test -f $(FPREFIX)3.lz4
# decompress multiple files : one output file per .lz4
- mv tmp-tlm1 tmp-tlm1-orig
- mv tmp-tlm2 tmp-tlm2-orig
- mv tmp-tlm3 tmp-tlm3-orig
- $(LZ4) -d -f -m tmp-tlm*.lz4
- $(CMP) tmp-tlm1 tmp-tlm1-orig # must be identical
- $(CMP) tmp-tlm2 tmp-tlm2-orig
- $(CMP) tmp-tlm3 tmp-tlm3-orig
+ mv $(FPREFIX)1 $(FPREFIX)1-orig
+ mv $(FPREFIX)2 $(FPREFIX)2-orig
+ mv $(FPREFIX)3 $(FPREFIX)3-orig
+ $(LZ4) -d -f -m $(FPREFIX)*.lz4
+ $(CMP) $(FPREFIX)1 $(FPREFIX)1-orig # must be identical
+ $(CMP) $(FPREFIX)2 $(FPREFIX)2-orig
+ $(CMP) $(FPREFIX)3 $(FPREFIX)3-orig
# compress multiple files into stdout
- $(CAT) tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 > tmp-tlm-concat1
- $(RM) *.lz4
- $(LZ4) -m tmp-tlm1 tmp-tlm2 tmp-tlm3 -c > tmp-tlm-concat2
- test ! -f tmp-tlm1.lz4 # must not create .lz4 artefact
- $(CMP) tmp-tlm-concat1 tmp-tlm-concat2 # must be equivalent
+ $(CAT) $(FPREFIX)1.lz4 $(FPREFIX)2.lz4 $(FPREFIX)3.lz4 > $(FPREFIX)-concat1
+ $(RM) $(FPREFIX)*.lz4
+ $(LZ4) -m $(FPREFIX)1 $(FPREFIX)2 $(FPREFIX)3 -c > $(FPREFIX)-concat2
+ test ! -f $(FPREFIX)1.lz4 # must not create .lz4 artefact
+ $(CMP) $(FPREFIX)-concat1 $(FPREFIX)-concat2 # must be equivalent
# decompress multiple files into stdout
- $(RM) tmp-tlm-concat1 tmp-tlm-concat2
- $(LZ4) -f -m tmp-tlm1 tmp-tlm2 tmp-tlm3 # generate .lz4 to decompress
- $(CAT) tmp-tlm1 tmp-tlm2 tmp-tlm3 > tmp-tlm-concat1 # create concatenated reference
- $(RM) tmp-tlm1 tmp-tlm2 tmp-tlm3
- $(LZ4) -d -m tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 -c > tmp-tlm-concat2
- test ! -f tmp-tlm1 # must not create file artefact
- $(CMP) tmp-tlm-concat1 tmp-tlm-concat2 # must be equivalent
+ $(RM) $(FPREFIX)-concat1 $(FPREFIX)-concat2
+ $(LZ4) -f -m $(FPREFIX)1 $(FPREFIX)2 $(FPREFIX)3 # generate .lz4 to decompress
+ $(CAT) $(FPREFIX)1 $(FPREFIX)2 $(FPREFIX)3 > $(FPREFIX)-concat1 # create concatenated reference
+ $(RM) $(FPREFIX)1 $(FPREFIX)2 $(FPREFIX)3
+ $(LZ4) -d -m $(FPREFIX)1.lz4 $(FPREFIX)2.lz4 $(FPREFIX)3.lz4 -c > $(FPREFIX)-concat2
+ test ! -f $(FPREFIX)1 # must not create file artefact
+ $(CMP) $(FPREFIX)-concat1 $(FPREFIX)-concat2 # must be equivalent
# compress multiple files, one of which is absent (must fail)
- ! $(LZ4) -f -m tmp-tlm-concat1 notHere tmp-tlm-concat2 # must fail : notHere not present
- @$(RM) tmp-tlm*
-
+ ! $(LZ4) -f -m $(FPREFIX)-concat1 notHere $(FPREFIX)-concat2 # must fail : notHere not present
+ # test lz4-compressed file
+ $(LZ4) -tm $(FPREFIX)-concat1.lz4
+ $(LZ4) -tm $(FPREFIX)-concat1.lz4 $(FPREFIX)-concat2.lz4
+ # test multiple lz4 files, one of which is absent (must fail)
+ ! $(LZ4) -tm $(FPREFIX)-concat1.lz4 notHere.lz4 $(FPREFIX)-concat2.lz4
+ @$(RM) $(FPREFIX)*
+
+test-lz4-multiple-legacy: FPREFIX = tmp-lml
test-lz4-multiple-legacy: lz4 datagen
@echo "\n ---- test multiple files (Legacy format) ----"
- @$(DATAGEN) -s1 > tmp-tlm1 2> $(VOID)
- @$(DATAGEN) -s2 -g100K > tmp-tlm2 2> $(VOID)
- @$(DATAGEN) -s3 -g200K > tmp-tlm3 2> $(VOID)
+ @$(DATAGEN) -s1 > $(FPREFIX)1 2> $(VOID)
+ @$(DATAGEN) -s2 -g100K > $(FPREFIX)2 2> $(VOID)
+ @$(DATAGEN) -s3 -g200K > $(FPREFIX)3 2> $(VOID)
# compress multiple files using legacy format: one .lz4 per source file
- $(LZ4) -f -l -m tmp-tlm*
- test -f tmp-tlm1.lz4
- test -f tmp-tlm2.lz4
- test -f tmp-tlm3.lz4
+ $(LZ4) -f -l -m $(FPREFIX)*
+ test -f $(FPREFIX)1.lz4
+ test -f $(FPREFIX)2.lz4
+ test -f $(FPREFIX)3.lz4
# decompress multiple files compressed using legacy format: one output file per .lz4
- mv tmp-tlm1 tmp-tlm1-orig
- mv tmp-tlm2 tmp-tlm2-orig
- mv tmp-tlm3 tmp-tlm3-orig
- $(LZ4) -d -f -m tmp-tlm*.lz4
- $(LZ4) -l -d -f -m tmp-tlm*.lz4 # -l mustn't impact -d option
- $(CMP) tmp-tlm1 tmp-tlm1-orig # must be identical
- $(CMP) tmp-tlm2 tmp-tlm2-orig
- $(CMP) tmp-tlm3 tmp-tlm3-orig
+ mv $(FPREFIX)1 $(FPREFIX)1-orig
+ mv $(FPREFIX)2 $(FPREFIX)2-orig
+ mv $(FPREFIX)3 $(FPREFIX)3-orig
+ $(LZ4) -d -f -m $(FPREFIX)*.lz4
+ $(LZ4) -l -d -f -m $(FPREFIX)*.lz4 # -l mustn't impact -d option
+ $(CMP) $(FPREFIX)1 $(FPREFIX)1-orig # must be identical
+ $(CMP) $(FPREFIX)2 $(FPREFIX)2-orig
+ $(CMP) $(FPREFIX)3 $(FPREFIX)3-orig
# compress multiple files into stdout using legacy format
- $(CAT) tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 > tmp-tlm-concat1
- $(RM) *.lz4
- $(LZ4) -l -m tmp-tlm1 tmp-tlm2 tmp-tlm3 -c > tmp-tlm-concat2
- test ! -f tmp-tlm1.lz4 # must not create .lz4 artefact
- $(CMP) tmp-tlm-concat1 tmp-tlm-concat2 # must be equivalent
+ $(CAT) $(FPREFIX)1.lz4 $(FPREFIX)2.lz4 $(FPREFIX)3.lz4 > $(FPREFIX)-concat1
+ $(RM) $(FPREFIX)*.lz4
+ $(LZ4) -l -m $(FPREFIX)1 $(FPREFIX)2 $(FPREFIX)3 -c > $(FPREFIX)-concat2
+ test ! -f $(FPREFIX)1.lz4 # must not create .lz4 artefact
+ $(CMP) $(FPREFIX)-concat1 $(FPREFIX)-concat2 # must be equivalent
# # # decompress multiple files into stdout using legacy format
- $(RM) tmp-tlm-concat1 tmp-tlm-concat2
- $(LZ4) -l -f -m tmp-tlm1 tmp-tlm2 tmp-tlm3 # generate .lz4 to decompress
- $(CAT) tmp-tlm1 tmp-tlm2 tmp-tlm3 > tmp-tlm-concat1 # create concatenated reference
- $(RM) tmp-tlm1 tmp-tlm2 tmp-tlm3
- $(LZ4) -d -m tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 -c > tmp-tlm-concat2
- $(LZ4) -d -l -m tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 -c > tmp-tlm-concat2 # -l mustn't impact option -d
- test ! -f tmp-tlm1 # must not create file artefact
- $(CMP) tmp-tlm-concat1 tmp-tlm-concat2 # must be equivalent
+ $(RM) $(FPREFIX)-concat1 $(FPREFIX)-concat2
+ $(LZ4) -l -f -m $(FPREFIX)1 $(FPREFIX)2 $(FPREFIX)3 # generate .lz4 to decompress
+ $(CAT) $(FPREFIX)1 $(FPREFIX)2 $(FPREFIX)3 > $(FPREFIX)-concat1 # create concatenated reference
+ $(RM) $(FPREFIX)1 $(FPREFIX)2 $(FPREFIX)3
+ $(LZ4) -d -m $(FPREFIX)1.lz4 $(FPREFIX)2.lz4 $(FPREFIX)3.lz4 -c > $(FPREFIX)-concat2
+ $(LZ4) -d -l -m $(FPREFIX)1.lz4 $(FPREFIX)2.lz4 $(FPREFIX)3.lz4 -c > $(FPREFIX)-concat2 # -l mustn't impact option -d
+ test ! -f $(FPREFIX)1 # must not create file artefact
+ $(CMP) $(FPREFIX)-concat1 $(FPREFIX)-concat2 # must be equivalent
# # # compress multiple files, one of which is absent (must fail)
- ! $(LZ4) -f -l -m tmp-tlm-concat1 notHere-legacy tmp-tlm-concat2 # must fail : notHere-legacy not present
- @$(RM) tmp-tlm*
-
+ ! $(LZ4) -f -l -m $(FPREFIX)-concat1 notHere-legacy $(FPREFIX)-concat2 # must fail : notHere-legacy not present
+ @$(RM) $(FPREFIX)*
+
+SKIPFILE = goldenSamples/skip.bin
+test-lz4-skippable: FPREFIX = tmp-lsk
+test-lz4-skippable: lz4 datagen
+ @echo "\n ---- test lz4 with skippable frames ----"
+ $(LZ4) -dc $(SKIPFILE)
+ $(LZ4) -dc < $(SKIPFILE)
+ cat $(SKIPFILE) | $(LZ4) -dc
+ echo "Hello from Valid Frame!\n" | $(LZ4) -c > $(FPREFIX).lz4
+ cat $(SKIPFILE) $(FPREFIX).lz4 $(SKIPFILE) | $(LZ4) -dc
+ $(RM) $(FPREFIX)*
+
+test-lz4-basic: FPREFIX = tmp-tlb
test-lz4-basic: lz4 datagen unlz4 lz4cat
@echo "\n ---- test lz4 basic compression/decompression ----"
$(DATAGEN) -g0 | $(LZ4) -v | $(LZ4) -t
$(DATAGEN) -g16KB | $(LZ4) -9 | $(LZ4) -t
- $(DATAGEN) -g20KB > tmp-tlb-dg20k
- $(LZ4) < tmp-tlb-dg20k | $(LZ4) -d > tmp-tlb-dec
- $(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec
- $(LZ4) --no-frame-crc < tmp-tlb-dg20k | $(LZ4) -d > tmp-tlb-dec
- $(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec
+ $(DATAGEN) -g20KB > $(FPREFIX)-dg20k
+ $(LZ4) < $(FPREFIX)-dg20k | $(LZ4) -d > $(FPREFIX)-dec
+ $(DIFF) -q $(FPREFIX)-dg20k $(FPREFIX)-dec
+ $(LZ4) --no-frame-crc < $(FPREFIX)-dg20k | $(LZ4) -d > $(FPREFIX)-dec
+ $(DIFF) -q $(FPREFIX)-dg20k $(FPREFIX)-dec
$(DATAGEN) | $(LZ4) -BI | $(LZ4) -t
+ $(DATAGEN) | $(LZ4) --no-crc | $(LZ4) -t
$(DATAGEN) -g6M -P99 | $(LZ4) -9BD | $(LZ4) -t
$(DATAGEN) -g17M | $(LZ4) -9v | $(LZ4) -qt
$(DATAGEN) -g33M | $(LZ4) --no-frame-crc | $(LZ4) -t
- $(DATAGEN) -g256MB | $(LZ4) -vqB4D | $(LZ4) -t
- @echo "hello world" > tmp-tlb-hw
- $(LZ4) --rm -f tmp-tlb-hw tmp-tlb-hw.lz4
- test ! -f tmp-tlb-hw # must fail (--rm)
- test -f tmp-tlb-hw.lz4
- $(PRGDIR)/lz4cat tmp-tlb-hw.lz4 # must display hello world
- test -f tmp-tlb-hw.lz4
- $(PRGDIR)/unlz4 --rm tmp-tlb-hw.lz4 tmp-tlb-hw
- test -f tmp-tlb-hw
- test ! -f tmp-tlb-hw.lz4 # must fail (--rm)
- test ! -f tmp-tlb-hw.lz4.lz4 # must fail (unlz4)
- $(PRGDIR)/lz4cat tmp-tlb-hw # pass-through mode
- test -f tmp-tlb-hw
- test ! -f tmp-tlb-hw.lz4 # must fail (lz4cat)
- $(LZ4) tmp-tlb-hw tmp-tlb-hw.lz4 # creates tmp-tlb-hw.lz4
- $(PRGDIR)/lz4cat < tmp-tlb-hw.lz4 > tmp-tlb3 # checks lz4cat works with stdin (#285)
- $(DIFF) -q tmp-tlb-hw tmp-tlb3
- $(PRGDIR)/lz4cat < tmp-tlb-hw > tmp-tlb2 # checks lz4cat works in pass-through mode
- $(DIFF) -q tmp-tlb-hw tmp-tlb2
- cp tmp-tlb-hw ./-d
+ $(DATAGEN) -g256MB | $(LZ4) -vqB4D | $(LZ4) -t --no-crc
+ @echo "hello world" > $(FPREFIX)-hw
+ $(LZ4) --rm -f $(FPREFIX)-hw $(FPREFIX)-hw.lz4
+ test ! -f $(FPREFIX)-hw # must fail (--rm)
+ test -f $(FPREFIX)-hw.lz4
+ $(PRGDIR)/lz4cat $(FPREFIX)-hw.lz4 | $(GREP) "hello world"
+ $(PRGDIR)/unlz4 --rm $(FPREFIX)-hw.lz4 $(FPREFIX)-hw
+ test -f $(FPREFIX)-hw
+ test ! -f $(FPREFIX)-hw.lz4 # must fail (--rm)
+ test ! -f $(FPREFIX)-hw.lz4.lz4 # must fail (unlz4)
+ $(PRGDIR)/lz4cat $(FPREFIX)-hw # pass-through mode
+ test -f $(FPREFIX)-hw
+ test ! -f $(FPREFIX)-hw.lz4 # must fail (lz4cat)
+ $(LZ4) $(FPREFIX)-hw $(FPREFIX)-hw.lz4 # creates $(FPREFIX)-hw.lz4
+ $(PRGDIR)/lz4cat < $(FPREFIX)-hw.lz4 > $(FPREFIX)3 # checks lz4cat works with stdin (#285)
+ $(DIFF) -q $(FPREFIX)-hw $(FPREFIX)3
+ $(PRGDIR)/lz4cat < $(FPREFIX)-hw > $(FPREFIX)2 # checks lz4cat works in pass-through mode
+ $(DIFF) -q $(FPREFIX)-hw $(FPREFIX)2
+ cp $(FPREFIX)-hw ./-d
$(LZ4) --rm -- -d -d.lz4 # compresses ./d into ./-d.lz4
test -f ./-d.lz4
test ! -f ./-d
mv ./-d.lz4 ./-z
- $(LZ4) -d --rm -- -z tmp-tlb4 # uncompresses ./-z into tmp-tlb4
+ $(LZ4) -d --rm -- -z $(FPREFIX)4 # uncompresses ./-z into $(FPREFIX)4
test ! -f ./-z
- $(DIFF) -q tmp-tlb-hw tmp-tlb4
- $(LZ4) -f tmp-tlb-hw
- $(LZ4) --list tmp-tlb-hw.lz4 # test --list on valid single-frame file
- $(CAT) tmp-tlb-hw >> tmp-tlb-hw.lz4
- $(LZ4) -f tmp-tlb-hw.lz4 # uncompress valid frame followed by invalid data
- $(LZ4) -BX tmp-tlb-hw -c -q | $(LZ4) -tv # test block checksum
+ $(DIFF) -q $(FPREFIX)-hw $(FPREFIX)4
+ ! $(LZ4) $(FPREFIX)2 $(FPREFIX)3 $(FPREFIX)4 # must fail: refuse to handle 3+ file names
+ $(LZ4) -f $(FPREFIX)-hw # create $(FPREFIX)-hw.lz4, for next tests
+ $(LZ4) --list $(FPREFIX)-hw.lz4 # test --list on valid single-frame file
+ $(LZ4) --list < $(FPREFIX)-hw.lz4 # test --list from stdin (file only)
+ $(CAT) $(FPREFIX)-hw >> $(FPREFIX)-hw.lz4
+ ! $(LZ4) -f $(FPREFIX)-hw.lz4 # uncompress valid frame followed by invalid data (must fail now)
+ $(LZ4) -BX $(FPREFIX)-hw -c -q | $(LZ4) -tv # test block checksum
# $(DATAGEN) -g20KB generates the same file every single time
# cannot save output of $(DATAGEN) -g20KB as input file to lz4 because the following shell commands are run before $(DATAGEN) -g20KB
test "$(shell $(DATAGEN) -g20KB | $(LZ4) -c --fast | wc -c)" -lt "$(shell $(DATAGEN) -g20KB | $(LZ4) -c --fast=9 | wc -c)" # -1 vs -9
test "$(shell $(DATAGEN) -g20KB | $(LZ4) -c -1 | wc -c)" -lt "$(shell $(DATAGEN) -g20KB| $(LZ4) -c --fast=1 | wc -c)" # 1 vs -1
test "$(shell $(DATAGEN) -g20KB | $(LZ4) -c --fast=1 | wc -c)" -eq "$(shell $(DATAGEN) -g20KB| $(LZ4) -c --fast| wc -c)" # checks default fast compression is -1
- ! $(LZ4) -c --fast=0 tmp-tlb-dg20K # lz4 should fail when fast=0
- ! $(LZ4) -c --fast=-1 tmp-tlb-dg20K # lz4 should fail when fast=-1
+ ! $(LZ4) -c --fast=0 $(FPREFIX)-dg20K # lz4 should fail when fast=0
+ ! $(LZ4) -c --fast=-1 $(FPREFIX)-dg20K # lz4 should fail when fast=-1
# High --fast values can result in out-of-bound dereferences #876
$(DATAGEN) -g1M | $(LZ4) -c --fast=999999999 > /dev/null
# Test for #596
- @echo "TEST" > tmp-tlb-test
- $(LZ4) -m tmp-tlb-test
- $(LZ4) tmp-tlb-test.lz4 tmp-tlb-test2
- $(DIFF) -q tmp-tlb-test tmp-tlb-test2
- @$(RM) tmp-tlb*
-
+ @echo "TEST" > $(FPREFIX)-test
+ $(LZ4) -m $(FPREFIX)-test
+ $(LZ4) $(FPREFIX)-test.lz4 $(FPREFIX)-test2
+ $(DIFF) -q $(FPREFIX)-test $(FPREFIX)-test2
+ @$(RM) $(FPREFIX)*
+test-lz4-dict: FPREFIX = tmp-dict
test-lz4-dict: lz4 datagen
@echo "\n ---- test lz4 compression/decompression with dictionary ----"
- $(DATAGEN) -g16KB > tmp-dict
- $(DATAGEN) -g32KB > tmp-dict-sample-32k
- < tmp-dict-sample-32k $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-32k
- $(DATAGEN) -g128MB > tmp-dict-sample-128m
- < tmp-dict-sample-128m $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-128m
- touch tmp-dict-sample-0
- < tmp-dict-sample-0 $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-0
+ $(DATAGEN) -g16KB > $(FPREFIX)
+ $(DATAGEN) -g32KB > $(FPREFIX)-sample-32k
+ < $(FPREFIX)-sample-32k $(LZ4) -D $(FPREFIX) | $(LZ4) -dD $(FPREFIX) | diff - $(FPREFIX)-sample-32k
+ $(DATAGEN) -g128MB > $(FPREFIX)-sample-128m
+ < $(FPREFIX)-sample-128m $(LZ4) -D $(FPREFIX) | $(LZ4) -dD $(FPREFIX) | diff - $(FPREFIX)-sample-128m
+ touch $(FPREFIX)-sample-0
+ < $(FPREFIX)-sample-0 $(LZ4) -D $(FPREFIX) | $(LZ4) -dD $(FPREFIX) | diff - $(FPREFIX)-sample-0
- < tmp-dict-sample-32k $(LZ4) -D tmp-dict-sample-0 | $(LZ4) -dD tmp-dict-sample-0 | diff - tmp-dict-sample-32k
- < tmp-dict-sample-0 $(LZ4) -D tmp-dict-sample-0 | $(LZ4) -dD tmp-dict-sample-0 | diff - tmp-dict-sample-0
+ < $(FPREFIX)-sample-32k $(LZ4) -D $(FPREFIX)-sample-0 | $(LZ4) -dD $(FPREFIX)-sample-0 | diff - $(FPREFIX)-sample-32k
+ < $(FPREFIX)-sample-0 $(LZ4) -D $(FPREFIX)-sample-0 | $(LZ4) -dD $(FPREFIX)-sample-0 | diff - $(FPREFIX)-sample-0
@echo "\n ---- test lz4 dictionary loading ----"
- $(DATAGEN) -g128KB > tmp-dict-data-128KB
+ $(DATAGEN) -g128KB > $(FPREFIX)-data-128KB
set -e; \
for l in 0 1 4 128 32767 32768 32769 65535 65536 65537 98303 98304 98305 131071 131072 131073; do \
- $(DATAGEN) -g$$l > tmp-dict-$$l; \
- $(DD) if=tmp-dict-$$l of=tmp-dict-$$l-tail bs=1 count=65536 skip=$$((l > 65536 ? l - 65536 : 0)); \
- < tmp-dict-$$l $(LZ4) -D stdin tmp-dict-data-128KB -c | $(LZ4) -dD tmp-dict-$$l-tail | $(DIFF) - tmp-dict-data-128KB; \
- < tmp-dict-$$l-tail $(LZ4) -D stdin tmp-dict-data-128KB -c | $(LZ4) -dD tmp-dict-$$l | $(DIFF) - tmp-dict-data-128KB; \
+ $(DATAGEN) -g$$l > $(FPREFIX)-$$l; \
+ $(DD) if=$(FPREFIX)-$$l of=$(FPREFIX)-$$l-tail bs=1 count=65536 skip=$$((l > 65536 ? l - 65536 : 0)); \
+ < $(FPREFIX)-$$l $(LZ4) -D stdin $(FPREFIX)-data-128KB -c | $(LZ4) -dD $(FPREFIX)-$$l-tail | $(DIFF) - $(FPREFIX)-data-128KB; \
+ < $(FPREFIX)-$$l-tail $(LZ4) -D stdin $(FPREFIX)-data-128KB -c | $(LZ4) -dD $(FPREFIX)-$$l | $(DIFF) - $(FPREFIX)-data-128KB; \
done
+ @$(RM) $(FPREFIX)*
- @$(RM) tmp-dict*
+test-lz4hc-hugefile: lz4 datagen
+ @echo "\n ---- test HC compression/decompression of huge files ----"
+ $(DATAGEN) -g4200MB | $(LZ4) -v3BD | $(LZ4) -qt
-test-lz4-hugefile: lz4 datagen
+test-lz4-fast-hugefile: FPREFIX = tmp-lfh
+test-lz4-fast-hugefile: lz4 datagen
@echo "\n ---- test huge files compression/decompression ----"
- ./datagen -g6GB | $(LZ4) -vB5D | $(LZ4) -qt
- ./datagen -g4500MB | $(LZ4) -v3BD | $(LZ4) -qt
+ $(DATAGEN) -g6GB | $(LZ4) -vB5D | $(LZ4) -qt
# test large file size [2-4] GB
- @$(DATAGEN) -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmphf1
- @ls -ls tmphf1
- @$(DATAGEN) -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmphf2
- @ls -ls tmphf2
- $(DIFF) -s tmphf1 tmphf2
- @$(RM) tmphf*
+ @$(DATAGEN) -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - $(FPREFIX)1
+ @ls -ls $(FPREFIX)1
+ @$(DATAGEN) -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - $(FPREFIX)2
+ @ls -ls $(FPREFIX)2
+ $(DIFF) -s $(FPREFIX)1 $(FPREFIX)2
+ @$(RM) $(FPREFIX)*
+
+test-lz4-hugefile: test-lz4-fast-hugefile test-lz4hc-hugefile
+test-lz4-testmode: FPREFIX = tmp-ltm
test-lz4-testmode: lz4 datagen
@echo "\n ---- bench mode ----"
$(LZ4) -bi0
+ $(DATAGEN) > $(FPREFIX)
+ $(LZ4) -f $(FPREFIX) -c > $(FPREFIX).lz4
+ $(LZ4) -bdi0 $(FPREFIX).lz4 # test benchmark decode-only mode
+ $(LZ4) -bdi0 --no-crc $(FPREFIX).lz4 # test benchmark decode-only mode
@echo "\n ---- test mode ----"
! $(DATAGEN) | $(LZ4) -t
! $(DATAGEN) | $(LZ4) -tf
@echo "\n ---- pass-through mode ----"
- @echo "Why hello there " > tmp-tlt2.lz4
- ! $(LZ4) -f tmp-tlt2.lz4 > $(VOID)
+ @echo "Why hello there " > $(FPREFIX)2.lz4
+ ! $(LZ4) -f $(FPREFIX)2.lz4 > $(VOID)
! $(DATAGEN) | $(LZ4) -dc > $(VOID)
! $(DATAGEN) | $(LZ4) -df > $(VOID)
$(DATAGEN) | $(LZ4) -dcf > $(VOID)
- @echo "Hello World !" > tmp-tlt1
- $(LZ4) -dcf tmp-tlt1
- @echo "from underground..." > tmp-tlt2
- $(LZ4) -dcfm tmp-tlt1 tmp-tlt2
- @echo "\n ---- non-existing source ----"
+ @echo "Hello World !" > $(FPREFIX)1
+ $(LZ4) -dcf $(FPREFIX)1
+ @echo "from underground..." > $(FPREFIX)2
+ $(LZ4) -dcfm $(FPREFIX)1 $(FPREFIX)2
+ @echo "\n ---- non-existing source (must fail cleanly) ----"
! $(LZ4) file-does-not-exist
! $(LZ4) -f file-does-not-exist
! $(LZ4) -t file-does-not-exist
! $(LZ4) -fm file1-dne file2-dne
- @$(RM) tmp-tlt tmp-tlt1 tmp-tlt2 tmp-tlt2.lz4
+ @$(RM) $(FPREFIX)*
test-lz4-opt-parser: lz4 datagen
@echo "\n ---- test opt-parser ----"
@@ -448,6 +511,7 @@ test-lz4-opt-parser: lz4 datagen
$(DATAGEN) -g256K | $(LZ4) -12B4D | $(LZ4) -t
$(DATAGEN) -g512K -P25 | $(LZ4) -12BD | $(LZ4) -t
$(DATAGEN) -g1M | $(LZ4) -12B5 | $(LZ4) -t
+ $(DATAGEN) -g1M -s2 | $(LZ4) -12B4D | $(LZ4) -t
$(DATAGEN) -g2M -P99 | $(LZ4) -11B4D | $(LZ4) -t
$(DATAGEN) -g4M | $(LZ4) -11vq | $(LZ4) -qt
$(DATAGEN) -g8M | $(LZ4) -11B4 | $(LZ4) -t
@@ -457,15 +521,15 @@ test-lz4-opt-parser: lz4 datagen
test-lz4-essentials : lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-multiple-legacy \
test-lz4-frame-concatenation test-lz4-testmode \
test-lz4-contentSize test-lz4-dict
- @$(RM) tmp*
test-lz4: lz4 datagen test-lz4-essentials test-lz4-opt-parser \
- test-lz4-sparse test-lz4-hugefile test-lz4-dict
- @$(RM) tmp*
+ test-lz4-sparse test-lz4-hugefile test-lz4-dict \
+ test-lz4-skippable
+test-lz4c: LZ4C = $(LZ4)c
test-lz4c: lz4c datagen
@echo "\n ---- test lz4c variant ----"
- $(DATAGEN) -g256MB | $(LZ4)c -l -v | $(LZ4)c -t
+ $(DATAGEN) -g256MB | $(LZ4C) -l -v | $(LZ4C) -t
test-lz4c32: CFLAGS+=-m32
test-lz4c32: test-lz4
@@ -514,31 +578,41 @@ test-frametest: frametest
test-frametest32: CFLAGS += -m32
test-frametest32: test-frametest
+VALGRIND = valgrind --leak-check=yes --error-exitcode=1
+test-mem: FPREFIX = tmp-tvm
test-mem: lz4 datagen fuzzer frametest fullbench
@echo "\n ---- valgrind tests : memory analyzer ----"
- valgrind --leak-check=yes --error-exitcode=1 $(DATAGEN) -g50M > $(VOID)
- $(DATAGEN) -g16KB > ftmdg16K
- valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -BD -f ftmdg16K $(VOID)
- $(DATAGEN) -g16KB -s2 > ftmdg16K2
- $(DATAGEN) -g16KB -s3 > ftmdg16K3
- valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --force --multiple ftmdg16K ftmdg16K2 ftmdg16K3
- $(DATAGEN) -g7MB > ftmdg7M
- valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f ftmdg7M ftmdg16K2
- valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -t ftmdg16K2
- valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 ftmdg7M
- valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 ftmdg7M ftmdg16K2
- valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq ftmdg7M $(VOID)
- valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --list -m ftm*.lz4
- valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --list -m -v ftm*.lz4
- $(RM) ftm*
- valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i64 -t1
- valgrind --leak-check=yes --error-exitcode=1 ./frametest -i256
+ $(VALGRIND) $(DATAGEN) -g50M > $(VOID)
+ $(DATAGEN) -g16KB > $(FPREFIX)dg16K
+ $(VALGRIND) $(LZ4) -9 -BD -f $(FPREFIX)dg16K $(VOID)
+ $(DATAGEN) -g16KB -s2 > $(FPREFIX)dg16K2
+ $(DATAGEN) -g16KB -s3 > $(FPREFIX)dg16K3
+ $(VALGRIND) $(LZ4) --force --multiple $(FPREFIX)dg16K $(FPREFIX)dg16K2 $(FPREFIX)dg16K3
+ $(DATAGEN) -g7MB > $(FPREFIX)dg7M
+ $(VALGRIND) $(LZ4) -9 -B5D -f $(FPREFIX)dg7M $(FPREFIX)dg16K2
+ $(VALGRIND) $(LZ4) -t $(FPREFIX)dg16K2
+ $(VALGRIND) $(LZ4) -bi1 $(FPREFIX)dg7M
+ $(VALGRIND) ./fullbench -i1 $(FPREFIX)dg7M $(FPREFIX)dg16K2
+ $(VALGRIND) $(LZ4) -B4D -f -vq $(FPREFIX)dg7M $(VOID)
+ $(VALGRIND) $(LZ4) --list -m $(FPREFIX)*.lz4
+ $(VALGRIND) $(LZ4) --list -m -v $(FPREFIX)*.lz4
+ $(RM) $(FPREFIX)*
+ $(VALGRIND) ./fuzzer -i64 -t1
+ $(VALGRIND) ./frametest -i256
test-mem32: lz4c32 datagen
# unfortunately, valgrind doesn't seem to work with non-native binary...
-test-decompress-partial : decompress-partial
+test-decompress-partial : decompress-partial decompress-partial-usingDict
@echo "\n ---- test decompress-partial ----"
./decompress-partial$(EXT)
+ @echo "\n ---- test decompress-partial-usingDict ----"
+ ./decompress-partial-usingDict$(EXT)
+
+test-freestanding: freestanding
+ @echo "\n ---- test freestanding ----"
+ ./freestanding$(EXT)
+ -strace ./freestanding$(EXT)
+ -ltrace ./freestanding$(EXT)
endif
diff --git a/tests/README.md b/tests/README.md
index 75b7b9f..65437de 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -13,7 +13,7 @@ This directory contains the following programs and scripts:
#### `test-lz4-versions.py` - script for testing lz4 interoperability between versions
This script creates `versionsTest` directory to which lz4 repository is cloned.
-Then all taged (released) versions of lz4 are compiled.
+Then all tagged (released) versions of lz4 are compiled.
In the following step interoperability between lz4 versions is checked.
@@ -25,7 +25,7 @@ After `sleepTime` (an optional parameter, default 300 seconds) seconds the scrip
If a new commit is found it is compiled and a speed benchmark for this commit is performed.
The results of the speed benchmark are compared to the previous results.
If compression or decompression speed for one of lz4 levels is lower than `lowerLimit` (an optional parameter, default 0.98) the speed benchmark is restarted.
-If second results are also lower than `lowerLimit` the warning e-mail is send to recipients from the list (the `emails` parameter).
+If second results are also lower than `lowerLimit` the warning e-mail is sent to recipients from the list (the `emails` parameter).
Additional remarks:
- To be sure that speed results are accurate the script should be run on a "stable" target system with no other jobs running in parallel
@@ -37,7 +37,7 @@ Additional remarks:
The example usage with two test files, one e-mail address, and with an additional message:
```
./test-lz4-speed.py "silesia.tar calgary.tar" "email@gmail.com" --message "tested on my laptop" --sleepTime 60
-```
+```
To run the script in background please use:
```
diff --git a/tests/abiTest.c b/tests/abiTest.c
new file mode 100644
index 0000000..e46004a
--- /dev/null
+++ b/tests/abiTest.c
@@ -0,0 +1,218 @@
+/*
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree),
+ * meaning you may select, at your option, one of the above-listed licenses.
+ */
+
+/*
+ * abiTest :
+ * ensure ABI stability expectations are not broken by a new version
+**/
+
+
+/*===========================================
+* Dependencies
+*==========================================*/
+#include <stddef.h> /* size_t */
+#include <stdlib.h> /* malloc, free, exit */
+#include <stdio.h> /* fprintf */
+#include <string.h> /* strcmp */
+#include <assert.h>
+#include <sys/types.h> /* stat */
+#include <sys/stat.h> /* stat */
+#include "xxhash.h"
+
+#include "lz4.h"
+#include "lz4frame.h"
+
+
+/*===========================================
+* Macros
+*==========================================*/
+#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
+
+#define MSG(...) fprintf(stderr, __VA_ARGS__)
+
+#define CONTROL_MSG(c, ...) { \
+ if ((c)) { \
+ MSG(__VA_ARGS__); \
+ MSG(" \n"); \
+ abort(); \
+ } \
+}
+
+
+static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize)
+{
+ const char* const ip1 = (const char*)buff1;
+ const char* const ip2 = (const char*)buff2;
+ size_t pos;
+
+ for (pos=0; pos<buffSize; pos++)
+ if (ip1[pos]!=ip2[pos])
+ break;
+
+ return pos;
+}
+
+
+LZ4_stream_t LZ4_cState;
+LZ4_streamDecode_t LZ4_dState;
+
+/** roundTripTest() :
+ * Compresses `srcBuff` into `compressedBuff`,
+ * then decompresses `compressedBuff` into `resultBuff`.
+ * If clevel==0, compression level is derived from srcBuff's content head bytes.
+ * This function abort() if it detects any round-trip error.
+ * Therefore, if it returns, round trip is considered successfully validated.
+ * Note : `compressedBuffCapacity` should be `>= LZ4_compressBound(srcSize)`
+ * for compression to be guaranteed to work */
+static void roundTripTest(void* resultBuff, size_t resultBuffCapacity,
+ void* compressedBuff, size_t compressedBuffCapacity,
+ const void* srcBuff, size_t srcSize)
+{
+ int const acceleration = 1;
+ // Note : can't use LZ4_initStream(), because it's only present since v1.9.0
+ memset(&LZ4_cState, 0, sizeof(LZ4_cState));
+ { int const cSize = LZ4_compress_fast_continue(&LZ4_cState, (const char*)srcBuff, (char*)compressedBuff, (int)srcSize, (int)compressedBuffCapacity, acceleration);
+ CONTROL_MSG(cSize == 0, "Compression error !");
+ { int const dInit = LZ4_setStreamDecode(&LZ4_dState, NULL, 0);
+ CONTROL_MSG(dInit == 0, "LZ4_setStreamDecode error !");
+ }
+ { int const dSize = LZ4_decompress_safe_continue (&LZ4_dState, (const char*)compressedBuff, (char*)resultBuff, cSize, (int)resultBuffCapacity);
+ CONTROL_MSG(dSize < 0, "Decompression detected an error !");
+ CONTROL_MSG(dSize != (int)srcSize, "Decompression corruption error : wrong decompressed size !");
+ } }
+
+ /* check potential content corruption error */
+ assert(resultBuffCapacity >= srcSize);
+ { size_t const errorPos = checkBuffers(srcBuff, resultBuff, srcSize);
+ CONTROL_MSG(errorPos != srcSize,
+ "Silent decoding corruption, at pos %u !!!",
+ (unsigned)errorPos);
+ }
+}
+
+static void roundTripCheck(const void* srcBuff, size_t srcSize)
+{
+ size_t const cBuffSize = LZ4_COMPRESSBOUND(srcSize);
+ void* const cBuff = malloc(cBuffSize);
+ void* const rBuff = malloc(cBuffSize);
+
+ if (!cBuff || !rBuff) {
+ fprintf(stderr, "not enough memory ! \n");
+ exit(1);
+ }
+
+ roundTripTest(rBuff, cBuffSize,
+ cBuff, cBuffSize,
+ srcBuff, srcSize);
+
+ free(rBuff);
+ free(cBuff);
+}
+
+
+static size_t getFileSize(const char* infilename)
+{
+ int r;
+#if defined(_MSC_VER)
+ struct _stat64 statbuf;
+ r = _stat64(infilename, &statbuf);
+ if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */
+#else
+ struct stat statbuf;
+ r = stat(infilename, &statbuf);
+ if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
+#endif
+ return (size_t)statbuf.st_size;
+}
+
+
+static int isDirectory(const char* infilename)
+{
+ int r;
+#if defined(_MSC_VER)
+ struct _stat64 statbuf;
+ r = _stat64(infilename, &statbuf);
+ if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
+#else
+ struct stat statbuf;
+ r = stat(infilename, &statbuf);
+ if (!r && S_ISDIR(statbuf.st_mode)) return 1;
+#endif
+ return 0;
+}
+
+
+/** loadFile() :
+ * requirement : `buffer` size >= `fileSize` */
+static void loadFile(void* buffer, const char* fileName, size_t fileSize)
+{
+ FILE* const f = fopen(fileName, "rb");
+ if (isDirectory(fileName)) {
+ MSG("Ignoring %s directory \n", fileName);
+ exit(2);
+ }
+ if (f==NULL) {
+ MSG("Impossible to open %s \n", fileName);
+ exit(3);
+ }
+ { size_t const readSize = fread(buffer, 1, fileSize, f);
+ if (readSize != fileSize) {
+ MSG("Error reading %s \n", fileName);
+ exit(5);
+ } }
+ fclose(f);
+}
+
+
+static void fileCheck(const char* fileName)
+{
+ size_t const fileSize = getFileSize(fileName);
+ void* const buffer = malloc(fileSize + !fileSize /* avoid 0 */);
+ if (!buffer) {
+ MSG("not enough memory \n");
+ exit(4);
+ }
+ loadFile(buffer, fileName, fileSize);
+ roundTripCheck(buffer, fileSize);
+ free (buffer);
+}
+
+
+int bad_usage(const char* exeName)
+{
+ MSG(" \n");
+ MSG("bad usage: \n");
+ MSG(" \n");
+ MSG("%s [Options] fileName \n", exeName);
+ MSG(" \n");
+ MSG("Options: \n");
+ MSG("-# : use #=[0-9] compression level (default:0 == random) \n");
+ return 1;
+}
+
+
+int main(int argCount, const char** argv)
+{
+ const char* const exeName = argv[0];
+ int argNb = 1;
+ // Note : LZ4_VERSION_STRING requires >= v1.7.3+
+ MSG("abiTest, built binary based on API %s \n", LZ4_VERSION_STRING);
+ // Note : LZ4_versionString() requires >= v1.7.5+
+ MSG("currently linked to dll %s \n", LZ4_versionString());
+
+ assert(argCount >= 1);
+ if (argCount < 2) return bad_usage(exeName);
+
+ if (argNb >= argCount) return bad_usage(exeName);
+
+ fileCheck(argv[argNb]);
+ MSG("no pb detected \n");
+ return 0;
+}
diff --git a/tests/checkFrame.c b/tests/checkFrame.c
index f9a1c14..946805f 100644
--- a/tests/checkFrame.c
+++ b/tests/checkFrame.c
@@ -1,6 +1,6 @@
/*
checkFrame - verify frame headers
- Copyright (C) Yann Collet 2014-present
+ Copyright (C) Yann Collet 2014-2020
GPL v2 License
@@ -153,7 +153,7 @@ int frameCheck(cRess_t ress, FILE* const srcFile, unsigned bsid, size_t blockSiz
if (LZ4F_isError(nextToLoad))
EXM_THROW(22, "Error getting frame info: %s",
LZ4F_getErrorName(nextToLoad));
- if (frameInfo.blockSizeID != bsid)
+ if (frameInfo.blockSizeID != (LZ4F_blockSizeID_t) bsid)
EXM_THROW(23, "Block size ID %u != expected %u",
frameInfo.blockSizeID, bsid);
pos += remaining;
diff --git a/tests/checkTag.c b/tests/checkTag.c
index 4a33415..5e5a034 100644
--- a/tests/checkTag.c
+++ b/tests/checkTag.c
@@ -1,6 +1,6 @@
/*
checkTag.c - Version validation tool for LZ4
- Copyright (C) Yann Collet 2018 - present
+ Copyright (C) Yann Collet 2018-2020
GPL v2 License
diff --git a/tests/check_liblz4_version.sh b/tests/check_liblz4_version.sh
new file mode 100755
index 0000000..9304204
--- /dev/null
+++ b/tests/check_liblz4_version.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env sh
+set -e
+
+# written as a script shell, because pipe management in python is horrible
+ldd $1 | grep liblz4
+
diff --git a/tests/datagencli.c b/tests/datagencli.c
index c985197..ccb27df 100644
--- a/tests/datagencli.c
+++ b/tests/datagencli.c
@@ -1,7 +1,7 @@
/*
datagencli.c
compressible data command line generator
- Copyright (C) Yann Collet 2012-2016
+ Copyright (C) Yann Collet 2012-2020
GPL v2 License
@@ -34,6 +34,14 @@
/**************************************
+* Compiler specific
+**************************************/
+#ifdef _MSC_VER /* Visual Studio */
+#define strtoull _strtoui64 /* https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/strtoui64-wcstoui64-strtoui64-l-wcstoui64-l */
+#endif
+
+
+/**************************************
* Constants
**************************************/
#define KB *(1 <<10)
@@ -103,13 +111,7 @@ int main(int argc, char** argv)
return usage(programName);
case 'g':
argument++;
- size=0;
- while ((*argument>='0') && (*argument<='9'))
- {
- size *= 10;
- size += *argument - '0';
- argument++;
- }
+ size = strtoull(argument, &argument, 10);
if (*argument=='K') { size <<= 10; argument++; }
if (*argument=='M') { size <<= 20; argument++; }
if (*argument=='G') { size <<= 30; argument++; }
@@ -117,35 +119,16 @@ int main(int argc, char** argv)
break;
case 's':
argument++;
- seed=0;
- while ((*argument>='0') && (*argument<='9'))
- {
- seed *= 10;
- seed += *argument - '0';
- argument++;
- }
+ seed = (U32) strtoul(argument, &argument, 10);
break;
case 'P':
argument++;
- proba=0.0;
- while ((*argument>='0') && (*argument<='9'))
- {
- proba *= 10;
- proba += *argument - '0';
- argument++;
- }
- if (proba>100.) proba=100.;
+ proba = (double) strtoull(argument, &argument, 10);
proba /= 100.;
break;
case 'L': /* hidden argument : Literal distribution probability */
argument++;
- litProba=0.;
- while ((*argument>='0') && (*argument<='9'))
- {
- litProba *= 10;
- litProba += *argument - '0';
- argument++;
- }
+ litProba = (double) strtoull(argument, &argument, 10);
if (litProba>100.) litProba=100.;
litProba /= 100.;
break;
diff --git a/tests/decompress-partial-usingDict.c b/tests/decompress-partial-usingDict.c
new file mode 100644
index 0000000..8b85106
--- /dev/null
+++ b/tests/decompress-partial-usingDict.c
@@ -0,0 +1,103 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <assert.h>
+#include "lz4.h"
+
+const char source[] =
+ "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod\n"
+ "tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim\n"
+ "veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea\n"
+ "commodo consequat. Duis aute irure dolor in reprehenderit in voluptate\n"
+ "velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat\n"
+ "cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id\n"
+ "est laborum.\n"
+ "\n"
+ "Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium\n"
+ "doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore\n"
+ "veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim\n"
+ "ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia\n"
+ "consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque\n"
+ "porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur,\n"
+ "adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore\n"
+ "et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis\n"
+ "nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid\n"
+ "ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea\n"
+ "voluptate velit esse quam nihil molestiae consequatur, vel illum qui\n"
+ "dolorem eum fugiat quo voluptas nulla pariatur?\n";
+
+#define BUFFER_SIZE 2048
+
+int main(void)
+{
+ int srcLen = (int)strlen(source);
+ size_t const smallSize = 1024;
+ size_t const largeSize = 64 * 1024 - 1;
+ char cmpBuffer[BUFFER_SIZE];
+ char* const buffer = (char*)malloc(BUFFER_SIZE + largeSize);
+ char* outBuffer = buffer + largeSize;
+ char* const dict = (char*)malloc(largeSize);
+ char* const largeDict = dict;
+ char* const smallDict = dict + largeSize - smallSize;
+ int i;
+ int cmpSize;
+
+ printf("starting test decompress-partial-usingDict : \n");
+ assert(buffer != NULL);
+ assert(dict != NULL);
+
+ cmpSize = LZ4_compress_default(source, cmpBuffer, srcLen, BUFFER_SIZE);
+
+ for (i = cmpSize; i < cmpSize + 10; ++i) {
+ int result = LZ4_decompress_safe_partial_usingDict(cmpBuffer, outBuffer, i, srcLen, BUFFER_SIZE, NULL, 0);
+ if ( (result < 0)
+ || (result != srcLen)
+ || memcmp(source, outBuffer, (size_t)srcLen) ) {
+ printf("test decompress-partial-usingDict with no dict error \n");
+ return -1;
+ }
+ }
+
+ for (i = cmpSize; i < cmpSize + 10; ++i) {
+ int result = LZ4_decompress_safe_partial_usingDict(cmpBuffer, outBuffer, i, srcLen, BUFFER_SIZE, outBuffer - smallSize, smallSize);
+ if ( (result < 0)
+ || (result != srcLen)
+ || memcmp(source, outBuffer, (size_t)srcLen) ) {
+ printf("test decompress-partial-usingDict with small prefix error \n");
+ return -1;
+ }
+ }
+
+ for (i = cmpSize; i < cmpSize + 10; ++i) {
+ int result = LZ4_decompress_safe_partial_usingDict(cmpBuffer, outBuffer, i, srcLen, BUFFER_SIZE, buffer, largeSize);
+ if ( (result < 0)
+ || (result != srcLen)
+ || memcmp(source, outBuffer, (size_t)srcLen) ) {
+ printf("test decompress-partial-usingDict with large prefix error \n");
+ return -1;
+ }
+ }
+
+ for (i = cmpSize; i < cmpSize + 10; ++i) {
+ int result = LZ4_decompress_safe_partial_usingDict(cmpBuffer, outBuffer, i, srcLen, BUFFER_SIZE, smallDict, smallSize);
+ if ( (result < 0)
+ || (result != srcLen)
+ || memcmp(source, outBuffer, (size_t)srcLen) ) {
+ printf("test decompress-partial-usingDict with small external dict error \n");
+ return -1;
+ }
+ }
+
+ for (i = cmpSize; i < cmpSize + 10; ++i) {
+ int result = LZ4_decompress_safe_partial_usingDict(cmpBuffer, outBuffer, i, srcLen, BUFFER_SIZE, largeDict, largeSize);
+ if ( (result < 0)
+ || (result != srcLen)
+ || memcmp(source, outBuffer, (size_t)srcLen) ) {
+ printf("test decompress-partial-usingDict with large external dict error \n");
+ return -1;
+ }
+ }
+
+ printf("test decompress-partial-usingDict OK \n");
+ return 0;
+}
diff --git a/tests/frametest.c b/tests/frametest.c
index e613cbf..3301955 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -1,6 +1,6 @@
/*
frameTest - test tool for lz4frame
- Copyright (C) Yann Collet 2014-2016
+ Copyright (C) Yann Collet 2014-2020
GPL v2 License
@@ -51,10 +51,10 @@
#include "xxhash.h" /* XXH64 */
-/* unoptimized version; solves endianess & alignment issues */
+/* unoptimized version; solves endianness & alignment issues */
static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32)
{
- BYTE* dstPtr = (BYTE*)dstVoidPtr;
+ BYTE* const dstPtr = (BYTE*)dstVoidPtr;
dstPtr[0] = (BYTE) value32;
dstPtr[1] = (BYTE)(value32 >> 8);
dstPtr[2] = (BYTE)(value32 >> 16);
@@ -65,8 +65,6 @@ static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32)
/*-************************************
* Constants
**************************************/
-#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U
-
#define KB *(1U<<10)
#define MB *(1U<<20)
#define GB *(1U<<30)
@@ -104,12 +102,63 @@ static U32 use_pause = 0;
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
#define MAX(a,b) ( (a) > (b) ? (a) : (b) )
+typedef struct {
+ int nbAllocs;
+} Test_alloc_state;
+static Test_alloc_state g_testAllocState = { 0 };
+
+static void* dummy_malloc(void* state, size_t s)
+{
+ Test_alloc_state* const t = (Test_alloc_state*)state;
+ void* const p = malloc(s);
+ if (p==NULL) return NULL;
+ assert(t != NULL);
+ t->nbAllocs += 1;
+ DISPLAYLEVEL(6, "Allocating %zu bytes at address %p \n", s, p);
+ DISPLAYLEVEL(5, "nb allocated memory segments : %i \n", t->nbAllocs);
+ return p;
+}
+
+static void* dummy_calloc(void* state, size_t s)
+{
+ Test_alloc_state* const t = (Test_alloc_state*)state;
+ void* const p = calloc(1, s);
+ if (p==NULL) return NULL;
+ assert(t != NULL);
+ t->nbAllocs += 1;
+ DISPLAYLEVEL(6, "Allocating and zeroing %zu bytes at address %p \n", s, p);
+ DISPLAYLEVEL(5, "nb allocated memory segments : %i \n", t->nbAllocs);
+ return p;
+}
+
+static void dummy_free(void* state, void* p)
+{
+ Test_alloc_state* const t = (Test_alloc_state*)state;
+ if (p==NULL) {
+ DISPLAYLEVEL(5, "free() on NULL \n");
+ return;
+ }
+ DISPLAYLEVEL(6, "freeing memory at address %p \n", p);
+ free(p);
+ assert(t != NULL);
+ t->nbAllocs -= 1;
+ DISPLAYLEVEL(5, "nb of allocated memory segments after this free : %i \n", t->nbAllocs);
+ assert(t->nbAllocs >= 0);
+}
+
+static const LZ4F_CustomMem lz4f_cmem_test = {
+ dummy_malloc,
+ dummy_calloc,
+ dummy_free,
+ &g_testAllocState
+};
+
+
static clock_t FUZ_GetClockSpan(clock_t clockStart)
{
return clock() - clockStart; /* works even if overflow; max span ~ 30 mn */
}
-
#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
unsigned int FUZ_rand(unsigned int* src)
{
@@ -121,7 +170,6 @@ unsigned int FUZ_rand(unsigned int* src)
return rand32 >> 5;
}
-
#define FUZ_RAND15BITS (FUZ_rand(seed) & 0x7FFF)
#define FUZ_RANDLENGTH ( (FUZ_rand(seed) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15)
static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, double proba, U32* seed)
@@ -515,7 +563,9 @@ int basicTests(U32 seed, double compressibility)
/* dictID tests */
{ size_t cErr;
U32 const dictID = 0x99;
- CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
+ /* test advanced variant with custom allocator functions */
+ cctx = LZ4F_createCompressionContext_advanced(lz4f_cmem_test, LZ4F_VERSION);
+ if (cctx==NULL) goto _output_error;
DISPLAYLEVEL(3, "insert a dictID : ");
memset(&prefs.frameInfo, 0, sizeof(prefs.frameInfo));
@@ -535,17 +585,25 @@ int basicTests(U32 seed, double compressibility)
}
/* Dictionary compression test */
- { size_t const dictSize = 63 KB;
- size_t const dstCapacity = LZ4F_compressFrameBound(dictSize, NULL);
+ { size_t const dictSize = 7 KB; /* small enough for LZ4_MEMORY_USAGE == 10 */
+ size_t const srcSize = 65 KB; /* must be > 64 KB to avoid short-size optimizations */
+ size_t const dstCapacity = LZ4F_compressFrameBound(srcSize, NULL);
size_t cSizeNoDict, cSizeWithDict;
LZ4F_CDict* const cdict = LZ4F_createCDict(CNBuffer, dictSize);
if (cdict == NULL) goto _output_error;
CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
+ DISPLAYLEVEL(3, "Testing LZ4F_createCDict_advanced : ");
+ { LZ4F_CDict* const cda = LZ4F_createCDict_advanced(lz4f_cmem_test, CNBuffer, dictSize);
+ if (cda == NULL) goto _output_error;
+ LZ4F_freeCDict(cda);
+ }
+ DISPLAYLEVEL(3, "OK \n");
+
DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with NULL dict : ");
CHECK_V(cSizeNoDict,
LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
- CNBuffer, dictSize,
+ CNBuffer, srcSize,
NULL, NULL) );
DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeNoDict);
@@ -554,16 +612,19 @@ int basicTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict : ");
CHECK_V(cSizeWithDict,
LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
- CNBuffer, dictSize,
+ CNBuffer, srcSize,
cdict, NULL) );
DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
- (unsigned)dictSize, (unsigned)cSizeWithDict);
- if ((LZ4_DISTANCE_MAX > dictSize) && (cSizeWithDict >= cSizeNoDict)) goto _output_error; /* must be more efficient */
- crcOrig = XXH64(CNBuffer, dictSize, 0);
+ (unsigned)srcSize, (unsigned)cSizeWithDict);
+ if (cSizeWithDict > cSizeNoDict) {
+ DISPLAYLEVEL(3, "cSizeWithDict (%zu) should have been more compact than cSizeNoDict(%zu) \n", cSizeWithDict, cSizeNoDict);
+ goto _output_error; /* must be more efficient */
+ }
+ crcOrig = XXH64(CNBuffer, srcSize, 0);
DISPLAYLEVEL(3, "LZ4F_decompress_usingDict : ");
{ LZ4F_dctx* dctx;
- size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
+ size_t decodedSize = srcSize;
size_t compressedSize = cSizeWithDict;
CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
CHECK( LZ4F_decompress_usingDict(dctx,
@@ -572,7 +633,7 @@ int basicTests(U32 seed, double compressibility)
CNBuffer, dictSize,
NULL) );
if (compressedSize != cSizeWithDict) goto _output_error;
- if (decodedSize != dictSize) goto _output_error;
+ if (decodedSize != srcSize) goto _output_error;
{ U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0);
if (crcDest != crcOrig) goto _output_error; }
DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
@@ -682,20 +743,20 @@ int basicTests(U32 seed, double compressibility)
{ size_t result;
unsigned blockSizeID;
for (blockSizeID = 4; blockSizeID < 8; ++blockSizeID) {
- result = LZ4F_getBlockSize(blockSizeID);
+ result = LZ4F_getBlockSize((LZ4F_blockSizeID_t)blockSizeID);
CHECK(result);
DISPLAYLEVEL(3, "Returned block size of %u bytes for blockID %u \n",
(unsigned)result, blockSizeID);
}
/* Test an invalid input that's too large */
- result = LZ4F_getBlockSize(8);
+ result = LZ4F_getBlockSize((LZ4F_blockSizeID_t)8);
if(!LZ4F_isError(result) ||
LZ4F_getErrorCode(result) != LZ4F_ERROR_maxBlockSize_invalid)
goto _output_error;
/* Test an invalid input that's too small */
- result = LZ4F_getBlockSize(3);
+ result = LZ4F_getBlockSize((LZ4F_blockSizeID_t)3);
if(!LZ4F_isError(result) ||
LZ4F_getErrorCode(result) != LZ4F_ERROR_maxBlockSize_invalid)
goto _output_error;
@@ -845,6 +906,7 @@ size_t test_lz4f_decompression_wBuffers(
memset(&dOptions, 0, sizeof(dOptions));
dOptions.stableDst = FUZ_rand(randState) & 1;
if (o_scenario == o_overwrite) dOptions.stableDst = 0; /* overwrite mode */
+ dOptions.skipChecksums = FUZ_rand(randState) & 127;
if (sentinelTest) op[oSizeMax] = mark;
DISPLAYLEVEL(7, "dstCapacity=%u, presentedInput=%u \n", (unsigned)oSize, (unsigned)iSize);
@@ -940,7 +1002,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
clock_t const startClock = clock();
clock_t const clockDuration = duration_s * CLOCKS_PER_SEC;
- /* Create buffers */
+ /* Create states & buffers */
{ size_t const creationStatus = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); }
{ size_t const creationStatus = LZ4F_createDecompressionContext(&dCtxNoise, LZ4F_VERSION);
@@ -1011,18 +1073,35 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
while (ip < iend) {
unsigned const nbBitsSeg = FUZ_rand(&randState) % maxBits;
size_t const sampleMax = (FUZ_rand(&randState) & ((1<<nbBitsSeg)-1)) + 1;
- size_t const iSize = MIN(sampleMax, (size_t)(iend-ip));
+ size_t iSize = MIN(sampleMax, (size_t)(iend-ip));
size_t const oSize = LZ4F_compressBound(iSize, prefsPtr);
- size_t flushedSize;
cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1);
DISPLAYLEVEL(6, "Sending %u bytes to compress (stableSrc:%u) \n",
(unsigned)iSize, cOptions.stableSrc);
- flushedSize = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions);
- CHECK(LZ4F_isError(flushedSize), "Compression failed (error %i : %s)",
+#if 1
+ /* insert uncompressed segment */
+ if ( (iSize>0)
+ && !neverFlush /* do not mess with compressBound when neverFlush is set */
+ && prefsPtr != NULL /* prefs are set */
+ && prefs.frameInfo.blockMode == LZ4F_blockIndependent /* uncompressedUpdate is only valid with blockMode==independent */
+ && (FUZ_rand(&randState) & 15) == 1 ) {
+ size_t const uSize = FUZ_rand(&randState) % iSize;
+ size_t const flushedSize = LZ4F_uncompressedUpdate(cCtx, op, (size_t)(oend-op), ip, uSize, &cOptions);
+ CHECK(LZ4F_isError(flushedSize), "Insert uncompressed data failed (error %i : %s)",
(int)flushedSize, LZ4F_getErrorName(flushedSize));
- op += flushedSize;
- ip += iSize;
+ op += flushedSize;
+ ip += uSize;
+ iSize -= uSize;
+ }
+#endif
+
+ { size_t const flushedSize = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions);
+ CHECK(LZ4F_isError(flushedSize), "Compression failed (error %i : %s)",
+ (int)flushedSize, LZ4F_getErrorName(flushedSize));
+ op += flushedSize;
+ ip += iSize;
+ }
{ unsigned const forceFlush = neverFlush ? 0 : ((FUZ_rand(&randState) & 3) == 1);
if (forceFlush) {
@@ -1036,11 +1115,8 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
op[3] = 0x80; /* 0x80000000U in little-endian format */
op += 4;
if ((prefsPtr!= NULL) && prefsPtr->frameInfo.blockChecksumFlag) {
- U32 const bc32 = XXH32(op, 0, 0);
- op[0] = (BYTE)bc32; /* little endian format */
- op[1] = (BYTE)(bc32>>8);
- op[2] = (BYTE)(bc32>>16);
- op[3] = (BYTE)(bc32>>24);
+ /* add block checksum (even for empty blocks) */
+ FUZ_writeLE32(op, XXH32(op, 0, 0));
op += 4;
} } } }
} /* while (ip<iend) */
diff --git a/tests/freestanding.c b/tests/freestanding.c
new file mode 100644
index 0000000..ceff4c5
--- /dev/null
+++ b/tests/freestanding.c
@@ -0,0 +1,239 @@
+// Basic test for LZ4_FREESTANDING
+
+// $ gcc -ffreestanding -nostdlib freestanding.c && ./a.out || echo $?
+
+// $ strace ./a.out
+// execve("./a.out", ["./a.out"], 0x7fffaf5fa580 /* 22 vars */) = 0
+// brk(NULL) = 0x56536f4fe000
+// arch_prctl(0x3001 /* ARCH_??? */, 0x7fffc9e74950) = -1 EINVAL (Invalid argument)
+// mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fd5c9c2b000
+// access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
+// arch_prctl(ARCH_SET_FS, 0x7fd5c9c2bc40) = 0
+// set_tid_address(0x7fd5c9c2bf10) = 381
+// set_robust_list(0x7fd5c9c2bf20, 24) = 0
+// rseq(0x7fd5c9c2c5e0, 0x20, 0, 0x53053053) = 0
+// mprotect(0x56536ea63000, 4096, PROT_READ) = 0
+// exit(0) = ?
+// +++ exited with 0 +++
+
+// $ ltrace ./a.out
+// +++ exited (status 0) +++
+
+#include <stddef.h>
+#include <stdint.h>
+
+#if defined(__cplusplus)
+# define EXTERN_C extern "C"
+#else
+# define EXTERN_C
+#endif
+
+
+#if !defined(__x86_64__) || !defined(__linux__)
+EXTERN_C void _start(void) { }
+int main(int argc, char** argv) { return 0; }
+#else
+
+static void MY_exit(int exitCode);
+static void MY_abort(void);
+EXTERN_C void *memmove(void *dst, const void *src, size_t n);
+EXTERN_C void *memcpy(void * __restrict__ dst, const void * __restrict__ src, size_t n);
+EXTERN_C void *memset(void *s, int c, size_t n);
+EXTERN_C int memcmp(const void *s1, const void *s2, size_t n);
+
+// LZ4/HC basic freestanding setup
+#define LZ4_FREESTANDING 1
+#define LZ4_memmove(dst, src, size) memmove((dst),(src),(size))
+#define LZ4_memcpy(dst, src, size) memcpy((dst),(src),(size))
+#define LZ4_memset(p,v,s) memset((p),(v),(s))
+
+#include "../lib/lz4.c"
+#include "../lib/lz4hc.c"
+
+// Test for LZ4
+static void test_lz4(const uint8_t* srcData, int srcSize) {
+ // Compress
+ static uint8_t compressBuffer[1024 * 1024];
+ const int compressedSize = LZ4_compress_default(
+ (const char*) srcData,
+ (char*) compressBuffer,
+ srcSize,
+ sizeof(compressBuffer)
+ );
+ if (compressedSize <= 0) {
+ MY_exit(__LINE__);
+ }
+
+ // Decompress
+ static uint8_t decompressBuffer[1024 * 1024];
+ const int decompressedSize = LZ4_decompress_safe(
+ (const char*) compressBuffer,
+ (char*) decompressBuffer,
+ compressedSize,
+ sizeof(decompressBuffer)
+ );
+ if (decompressedSize <= 0) {
+ MY_exit(__LINE__);
+ }
+
+ // Verify
+ if (decompressedSize != srcSize) {
+ MY_exit(__LINE__);
+ }
+ if (memcmp(srcData, decompressBuffer, srcSize) != 0) {
+ MY_exit(__LINE__);
+ }
+}
+
+
+// Test for LZ4HC
+static void test_lz4hc(const uint8_t* srcData, int srcSize) {
+ // Compress
+ static uint8_t compressBuffer[1024 * 1024];
+ const int compressedSize = LZ4_compress_HC(
+ (const char*) srcData,
+ (char*) compressBuffer,
+ srcSize,
+ sizeof(compressBuffer),
+ LZ4HC_CLEVEL_DEFAULT
+ );
+ if (compressedSize <= 0) {
+ MY_exit(__LINE__);
+ }
+
+ // Decompress
+ static uint8_t decompressBuffer[1024 * 1024];
+ const int decompressedSize = LZ4_decompress_safe(
+ (const char*) compressBuffer,
+ (char*) decompressBuffer,
+ compressedSize,
+ sizeof(decompressBuffer)
+ );
+ if (decompressedSize <= 0) {
+ MY_exit(__LINE__);
+ }
+
+ // Verify
+ if (decompressedSize != srcSize) {
+ MY_exit(__LINE__);
+ }
+ if (memcmp(srcData, decompressBuffer, srcSize) != 0) {
+ MY_exit(__LINE__);
+ }
+}
+
+
+static void test(void) {
+ // First 256 bytes of lz4/README.md
+ static const uint8_t README_md[] = {
+ 0x4c, 0x5a, 0x34, 0x20, 0x2d, 0x20, 0x45, 0x78, 0x74, 0x72, 0x65, 0x6d, 0x65, 0x6c, 0x79, 0x20,
+ 0x66, 0x61, 0x73, 0x74, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e,
+ 0x0a, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d,
+ 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d, 0x3d,
+ 0x3d, 0x0a, 0x0a, 0x4c, 0x5a, 0x34, 0x20, 0x69, 0x73, 0x20, 0x6c, 0x6f, 0x73, 0x73, 0x6c, 0x65,
+ 0x73, 0x73, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x20, 0x61,
+ 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x2c, 0x0a, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64,
+ 0x69, 0x6e, 0x67, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x20,
+ 0x73, 0x70, 0x65, 0x65, 0x64, 0x20, 0x3e, 0x20, 0x35, 0x30, 0x30, 0x20, 0x4d, 0x42, 0x2f, 0x73,
+ 0x20, 0x70, 0x65, 0x72, 0x20, 0x63, 0x6f, 0x72, 0x65, 0x2c, 0x0a, 0x73, 0x63, 0x61, 0x6c, 0x61,
+ 0x62, 0x6c, 0x65, 0x20, 0x77, 0x69, 0x74, 0x68, 0x20, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x2d, 0x63,
+ 0x6f, 0x72, 0x65, 0x73, 0x20, 0x43, 0x50, 0x55, 0x2e, 0x0a, 0x49, 0x74, 0x20, 0x66, 0x65, 0x61,
+ 0x74, 0x75, 0x72, 0x65, 0x73, 0x20, 0x61, 0x6e, 0x20, 0x65, 0x78, 0x74, 0x72, 0x65, 0x6d, 0x65,
+ 0x6c, 0x79, 0x20, 0x66, 0x61, 0x73, 0x74, 0x20, 0x64, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2c,
+ 0x0a, 0x77, 0x69, 0x74, 0x68, 0x20, 0x73, 0x70, 0x65, 0x65, 0x64, 0x20, 0x69, 0x6e, 0x20, 0x6d,
+ 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, 0x65, 0x20, 0x47, 0x42, 0x2f, 0x73, 0x20, 0x70, 0x65, 0x72,
+ };
+
+ static const uint8_t* srcData = README_md;
+ static const int srcSize = (int) sizeof(README_md);
+ test_lz4 (srcData, srcSize);
+ test_lz4hc(srcData, srcSize);
+}
+
+
+// low level syscall
+#define SYS_exit (60)
+
+static __inline long os_syscall1(long n, long a1) {
+ register long rax __asm__ ("rax") = n;
+ register long rdi __asm__ ("rdi") = a1;
+ __asm__ __volatile__ ("syscall" : "+r"(rax) : "r"(rdi) : "rcx", "r11", "memory");
+ return rax;
+}
+
+static void MY_exit(int exitCode) {
+ (void) os_syscall1(SYS_exit, exitCode);
+ __builtin_unreachable(); // suppress "warning: 'noreturn' function does return"
+}
+
+static void MY_abort(void) {
+ MY_exit(-1);
+}
+
+// https://refspecs.linuxbase.org/LSB_3.0.0/LSB-Core-generic/LSB-Core-generic/baselib---assert-fail-1.html
+void __assert_fail(const char * assertion, const char * file, unsigned int line, const char * function) {
+ MY_abort();
+}
+
+
+// GCC requires memcpy, memmove, memset and memcmp.
+// https://gcc.gnu.org/onlinedocs/gcc/Standards.html
+// > GCC requires the freestanding environment provide memcpy, memmove, memset and memcmp.
+EXTERN_C void *memmove(void *dst, const void *src, size_t n) {
+ uint8_t* d = (uint8_t*) dst;
+ const uint8_t* s = (const uint8_t*) src;
+
+ if (d > s) {
+ d += n;
+ s += n;
+ while (n--) {
+ *--d = *--s;
+ }
+ } else {
+ while (n--) {
+ *d++ = *s++;
+ }
+ }
+ return dst;
+}
+
+EXTERN_C void *memcpy(void * __restrict__ dst, const void * __restrict__ src, size_t n) {
+ return memmove(dst, src, n);
+}
+
+EXTERN_C void *memset(void *s, int c, size_t n) {
+ uint8_t* p = (uint8_t*) s;
+ while (n--) {
+ *p++ = (uint8_t) c;
+ }
+ return s;
+}
+
+EXTERN_C int memcmp(const void *s1, const void *s2, size_t n) {
+ const uint8_t* p1 = (const uint8_t*) s1;
+ const uint8_t* p2 = (const uint8_t*) s2;
+ while (n--) {
+ const uint8_t c1 = *p1++;
+ const uint8_t c2 = *p2++;
+ if (c1 < c2) {
+ return -1;
+ } else if (c1 > c2) {
+ return 1;
+ }
+ }
+ return 0;
+}
+
+
+//
+EXTERN_C void _start(void) {
+ test();
+ MY_exit(0);
+}
+
+int main(int argc, char** argv) {
+ test();
+ MY_exit(0);
+ return 0;
+}
+#endif
diff --git a/tests/fullbench.c b/tests/fullbench.c
index cb9b684..9c13996 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -1,6 +1,6 @@
/*
bench.c - Demo program to benchmark open-source compression algorithm
- Copyright (C) Yann Collet 2012-2016
+ Copyright (C) Yann Collet 2012-2020
GPL v2 License
@@ -312,6 +312,13 @@ static int local_LZ4_decompress_safe_usingDict(const char* in, char* out, int in
return outSize;
}
+static int local_LZ4_decompress_safe_partial_usingDict(const char* in, char* out, int inSize, int outSize)
+{
+ int result = LZ4_decompress_safe_partial_usingDict(in, out, inSize, outSize - 5, outSize, out - 65536, 65536);
+ if (result < 0) return result;
+ return outSize;
+}
+
#ifndef LZ4_DLL_IMPORT
#if defined (__cplusplus)
extern "C" {
@@ -325,12 +332,30 @@ extern int LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSiz
static int local_LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize)
{
- (void)inSize;
LZ4_decompress_safe_forceExtDict(in, out, inSize, outSize, out - 65536, 65536);
return outSize;
}
#endif
+#ifndef LZ4_DLL_IMPORT
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+extern int LZ4_decompress_safe_partial_forceExtDict(const char* in, char* out, int inSize, int targetOutputSize, int dstCapacity, const void* dict, size_t dictSize);
+
+#if defined (__cplusplus)
+}
+#endif
+
+static int local_LZ4_decompress_safe_partial_forceExtDict(const char* in, char* out, int inSize, int outSize)
+{
+ int result = LZ4_decompress_safe_partial_forceExtDict(in, out, inSize, outSize - 5, outSize, out - 65536, 65536);
+ if (result < 0) return result;
+ return outSize;
+}
+#endif
+
static int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSize, int outSize)
{
int result = LZ4_decompress_safe_partial(in, out, inSize, outSize - 5, outSize);
@@ -372,7 +397,8 @@ static int local_LZ4F_decompress_followHint(const char* src, char* dst, int srcS
size_t outRemaining = maxOutSize - outPos;
for (;;) {
- size_t const sizeHint = LZ4F_decompress(g_dCtx, dst+outPos, &outRemaining, src+inPos, &inSize, NULL);
+ size_t const sizeHint =
+ LZ4F_decompress(g_dCtx, dst+outPos, &outRemaining, src+inPos, &inSize, NULL);
assert(!LZ4F_isError(sizeHint));
inPos += inSize;
@@ -657,15 +683,17 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
case 5: decompressionFunction = local_LZ4_decompress_safe_withPrefix64k; dName = "LZ4_decompress_safe_withPrefix64k"; break;
case 6: decompressionFunction = local_LZ4_decompress_safe_usingDict; dName = "LZ4_decompress_safe_usingDict"; break;
case 7: decompressionFunction = local_LZ4_decompress_safe_partial; dName = "LZ4_decompress_safe_partial"; checkResult = 0; break;
+ case 8: decompressionFunction = local_LZ4_decompress_safe_partial_usingDict; dName = "LZ4_decompress_safe_partial_usingDict"; checkResult = 0; break;
#ifndef LZ4_DLL_IMPORT
- case 8: decompressionFunction = local_LZ4_decompress_safe_forceExtDict; dName = "LZ4_decompress_safe_forceExtDict"; break;
+ case 9: decompressionFunction = local_LZ4_decompress_safe_partial_forceExtDict; dName = "LZ4_decompress_safe_partial_forceExtDict"; checkResult = 0; break;
+ case 10: decompressionFunction = local_LZ4_decompress_safe_forceExtDict; dName = "LZ4_decompress_safe_forceExtDict"; break;
#endif
- case 10:
case 11:
case 12:
- if (dAlgNb == 10) { decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress"; } /* can be skipped */
- if (dAlgNb == 11) { decompressionFunction = local_LZ4F_decompress_followHint; dName = "LZ4F_decompress_followHint"; } /* can be skipped */
- if (dAlgNb == 12) { decompressionFunction = local_LZ4F_decompress_noHint; dName = "LZ4F_decompress_noHint"; } /* can be skipped */
+ case 13:
+ if (dAlgNb == 11) { decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress"; } /* can be skipped */
+ if (dAlgNb == 12) { decompressionFunction = local_LZ4F_decompress_followHint; dName = "LZ4F_decompress_followHint"; } /* can be skipped */
+ if (dAlgNb == 13) { decompressionFunction = local_LZ4F_decompress_noHint; dName = "LZ4F_decompress_noHint"; } /* can be skipped */
/* prepare compressed data using frame format */
{ size_t const fcsize = LZ4F_compressFrame(compressed_buff, (size_t)compressedBuffSize, orig_buff, benchedSize, NULL);
assert(!LZ4F_isError(fcsize));
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index a824813..341a2b0 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1,6 +1,6 @@
/*
fuzzer.c - Fuzzer test tool for LZ4
- Copyright (C) Yann Collet 2012-2017
+ Copyright (C) Yann Collet 2012-2020
GPL v2 License
@@ -30,6 +30,7 @@
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
# pragma warning(disable : 4310) /* disable: C4310: constant char value > 127 */
+# pragma warning(disable : 26451) /* disable: C26451: Arithmetic overflow */
#endif
@@ -494,7 +495,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
{ char* const cBuffer_exact = (char*)malloc((size_t)compressedSize);
assert(cBuffer_exact != NULL);
assert(compressedSize <= (int)compressedBufferSize);
+#if defined(_MSC_VER) && (_MSC_VER <= 1933) /* MSVC 2022 ver 17.3 or earlier */
+# pragma warning(push)
+# pragma warning(disable : 6385) /* lz4\tests\fuzzer.c(497): warning C6385: Reading invalid data from 'compressedBuffer'. */
+#endif
memcpy(cBuffer_exact, compressedBuffer, compressedSize);
+#if defined(_MSC_VER) && (_MSC_VER <= 1933) /* MSVC 2022 ver 17.3 or earlier */
+# pragma warning(pop)
+#endif
/* Test decoding with output size exactly correct => must work */
FUZ_DISPLAYTEST("LZ4_decompress_fast() with exact output buffer");
@@ -571,7 +579,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
for (;;) {
/* keep some original src */
{ U32 const nbBits = FUZ_rand(&randState) % maxNbBits;
- size_t const mask = (1<<nbBits) - 1;
+ size_t const mask = (1ULL <<nbBits) - 1;
size_t const skipLength = FUZ_rand(&randState) & mask;
pos += skipLength;
}
@@ -579,7 +587,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
/* add noise */
{ U32 const nbBitsCodes = FUZ_rand(&randState) % maxNbBits;
U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0;
- size_t const mask = (1<<nbBits) - 1;
+ size_t const mask = (1ULL <<nbBits) - 1;
size_t const rNoiseLength = (FUZ_rand(&randState) & mask) + 1;
size_t const noiseLength = MIN(rNoiseLength, (size_t)compressedSize-pos);
size_t const noiseStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - noiseLength);
@@ -630,6 +638,46 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(memcmp(block, decodedBuffer, (size_t)targetSize), "LZ4_decompress_safe_partial: corruption detected in regenerated data");
}
+ /* Partial decompression using dictionary. */
+ FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial_usingDict using no dict");
+ { size_t const missingOutBytes = FUZ_rand(&randState) % (unsigned)blockSize;
+ int const targetSize = (int)((size_t)blockSize - missingOutBytes);
+ size_t const extraneousInBytes = FUZ_rand(&randState) % 2;
+ int const inCSize = (int)((size_t)compressedSize + extraneousInBytes);
+ char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A;
+ int const decResult = LZ4_decompress_safe_partial_usingDict(compressedBuffer, decodedBuffer, inCSize, targetSize, blockSize, NULL, 0);
+ FUZ_CHECKTEST(decResult<0, "LZ4_decompress_safe_partial_usingDict failed despite valid input data (error:%i)", decResult);
+ FUZ_CHECKTEST(decResult != targetSize, "LZ4_decompress_safe_partial_usingDict did not regenerated required amount of data (%i < %i <= %i)", decResult, targetSize, blockSize);
+ FUZ_CHECKTEST(decodedBuffer[targetSize] != sentinel, "LZ4_decompress_safe_partial_usingDict overwrite beyond requested size (though %i <= %i <= %i)", decResult, targetSize, blockSize);
+ FUZ_CHECKTEST(memcmp(block, decodedBuffer, (size_t)targetSize), "LZ4_decompress_safe_partial_usingDict: corruption detected in regenerated data");
+ }
+
+ FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial_usingDict() using prefix as dict");
+ { size_t const missingOutBytes = FUZ_rand(&randState) % (unsigned)blockSize;
+ int const targetSize = (int)((size_t)blockSize - missingOutBytes);
+ size_t const extraneousInBytes = FUZ_rand(&randState) % 2;
+ int const inCSize = (int)((size_t)compressedSize + extraneousInBytes);
+ char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A;
+ int const decResult = LZ4_decompress_safe_partial_usingDict(compressedBuffer, decodedBuffer, inCSize, targetSize, blockSize, decodedBuffer, dictSize);
+ FUZ_CHECKTEST(decResult<0, "LZ4_decompress_safe_partial_usingDict failed despite valid input data (error:%i)", decResult);
+ FUZ_CHECKTEST(decResult != targetSize, "LZ4_decompress_safe_partial_usingDict did not regenerated required amount of data (%i < %i <= %i)", decResult, targetSize, blockSize);
+ FUZ_CHECKTEST(decodedBuffer[targetSize] != sentinel, "LZ4_decompress_safe_partial_usingDict overwrite beyond requested size (though %i <= %i <= %i)", decResult, targetSize, blockSize);
+ FUZ_CHECKTEST(memcmp(block, decodedBuffer, (size_t)targetSize), "LZ4_decompress_safe_partial_usingDict: corruption detected in regenerated data");
+ }
+
+ FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial_usingDict() using external dict");
+ { size_t const missingOutBytes = FUZ_rand(&randState) % (unsigned)blockSize;
+ int const targetSize = (int)((size_t)blockSize - missingOutBytes);
+ size_t const extraneousInBytes = FUZ_rand(&randState) % 2;
+ int const inCSize = (int)((size_t)compressedSize + extraneousInBytes);
+ char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A;
+ int const decResult = LZ4_decompress_safe_partial_usingDict(compressedBuffer, decodedBuffer, inCSize, targetSize, blockSize, dict, dictSize);
+ FUZ_CHECKTEST(decResult<0, "LZ4_decompress_safe_partial_usingDict failed despite valid input data (error:%i)", decResult);
+ FUZ_CHECKTEST(decResult != targetSize, "LZ4_decompress_safe_partial_usingDict did not regenerated required amount of data (%i < %i <= %i)", decResult, targetSize, blockSize);
+ FUZ_CHECKTEST(decodedBuffer[targetSize] != sentinel, "LZ4_decompress_safe_partial_usingDict overwrite beyond requested size (though %i <= %i <= %i)", decResult, targetSize, blockSize);
+ FUZ_CHECKTEST(memcmp(block, decodedBuffer, (size_t)targetSize), "LZ4_decompress_safe_partial_usingDict: corruption detected in regenerated data");
+ }
+
/* Test Compression with limited output size */
/* Test compression with output size being exactly what's necessary (should work) */
@@ -1136,12 +1184,12 @@ static void FUZ_unitTests(int compressionLevel)
assert(shc != NULL);
memset(shc, 0, sizeof(*shc));
DISPLAYLEVEL(4, "state1(%p) state2(%p) state3(%p) LZ4_stream_t size(0x%x): ",
- &(shc->state1), &(shc->state2), &(shc->state3), (unsigned)sizeof(LZ4_stream_t));
- FUZ_CHECKTEST( LZ4_initStream(&(shc->state1), sizeof(shc->state1)) == NULL, "state1 (%p) failed init", &(shc->state1) );
- FUZ_CHECKTEST( LZ4_initStream(&(shc->state2), sizeof(shc->state2)) == NULL, "state2 (%p) failed init", &(shc->state2) );
- FUZ_CHECKTEST( LZ4_initStream(&(shc->state3), sizeof(shc->state3)) == NULL, "state3 (%p) failed init", &(shc->state3) );
+ (void*)&(shc->state1), (void*)&(shc->state2), (void*)&(shc->state3), (unsigned)sizeof(LZ4_stream_t));
+ FUZ_CHECKTEST( LZ4_initStream(&(shc->state1), sizeof(shc->state1)) == NULL, "state1 (%p) failed init", (void*)&(shc->state1) );
+ FUZ_CHECKTEST( LZ4_initStream(&(shc->state2), sizeof(shc->state2)) == NULL, "state2 (%p) failed init", (void*)&(shc->state2) );
+ FUZ_CHECKTEST( LZ4_initStream(&(shc->state3), sizeof(shc->state3)) == NULL, "state3 (%p) failed init", (void*)&(shc->state3) );
FUZ_CHECKTEST( LZ4_initStream((char*)&(shc->state1) + 1, sizeof(shc->state1)) != NULL,
- "hc1+1 (%p) init must fail, due to bad alignment", (char*)&(shc->state1) + 1 );
+ "hc1+1 (%p) init must fail, due to bad alignment", (void*)((char*)&(shc->state1) + 1) );
free(shc);
}
DISPLAYLEVEL(3, "all inits OK \n");
@@ -1246,12 +1294,13 @@ static void FUZ_unitTests(int compressionLevel)
assert(shc != NULL);
memset(shc, 0, sizeof(*shc));
DISPLAYLEVEL(4, "hc1(%p) hc2(%p) hc3(%p) size(0x%x): ",
- &(shc->hc1), &(shc->hc2), &(shc->hc3), (unsigned)sizeof(LZ4_streamHC_t));
- FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc1), sizeof(shc->hc1)) == NULL, "hc1 (%p) failed init", &(shc->hc1) );
- FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc2), sizeof(shc->hc2)) == NULL, "hc2 (%p) failed init", &(shc->hc2) );
- FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc3), sizeof(shc->hc3)) == NULL, "hc3 (%p) failed init", &(shc->hc3) );
+ (void*)&(shc->hc1), (void*)&(shc->hc2), (void*)&(shc->hc3),
+ (unsigned)sizeof(LZ4_streamHC_t));
+ FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc1), sizeof(shc->hc1)) == NULL, "hc1 (%p) failed init", (void*)&(shc->hc1) );
+ FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc2), sizeof(shc->hc2)) == NULL, "hc2 (%p) failed init", (void*)&(shc->hc2) );
+ FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc3), sizeof(shc->hc3)) == NULL, "hc3 (%p) failed init", (void*)&(shc->hc3) );
FUZ_CHECKTEST( LZ4_initStreamHC((char*)&(shc->hc1) + 1, sizeof(shc->hc1)) != NULL,
- "hc1+1 (%p) init must fail, due to bad alignment", (char*)&(shc->hc1) + 1 );
+ "hc1+1 (%p) init must fail, due to bad alignment", (void*)((char*)&(shc->hc1) + 1) );
free(shc);
}
DISPLAYLEVEL(3, "all inits OK \n");
diff --git a/tests/goldenSamples/skip.bin b/tests/goldenSamples/skip.bin
new file mode 100644
index 0000000..1a8b9d5
--- /dev/null
+++ b/tests/goldenSamples/skip.bin
Binary files differ
diff --git a/tests/roundTripTest.c b/tests/roundTripTest.c
index 2d34451..3e9d6ed 100644
--- a/tests/roundTripTest.c
+++ b/tests/roundTripTest.c
@@ -1,11 +1,11 @@
/*
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
+ * in the COPYING file in the root directory of this source tree),
+ * meaning you may select, at your option, one of the above-listed licenses.
*/
/*
@@ -104,7 +104,7 @@ static void roundTripTest(void* resultBuff, size_t resultBuffCapacity,
int clevel)
{
int const proposed_clevel = clevel ? clevel : select_clevel(srcBuff, srcSize);
- int const selected_clevel = proposed_clevel < 0 ? -proposed_clevel : proposed_clevel; /* if level < 0, it becomes an accelearion value */
+ int const selected_clevel = proposed_clevel < 0 ? -proposed_clevel : proposed_clevel; /* if level < 0, it becomes an acceleration value */
compressFn compress = selected_clevel >= LZ4HC_CLEVEL_MIN ? LZ4_compress_HC : LZ4_compress_fast;
int const cSize = compress((const char*)srcBuff, (char*)compressedBuff, (int)srcSize, (int)compressedBuffCapacity, selected_clevel);
CONTROL_MSG(cSize == 0, "Compression error !");
@@ -126,7 +126,7 @@ static void roundTripTest(void* resultBuff, size_t resultBuffCapacity,
static void roundTripCheck(const void* srcBuff, size_t srcSize, int clevel)
{
- size_t const cBuffSize = LZ4_compressBound((int)srcSize);
+ size_t const cBuffSize = LZ4_COMPRESSBOUND(srcSize);
void* const cBuff = malloc(cBuffSize);
void* const rBuff = malloc(cBuffSize);
diff --git a/tests/test-lz4-abi.py b/tests/test-lz4-abi.py
new file mode 100644
index 0000000..0c8fd05
--- /dev/null
+++ b/tests/test-lz4-abi.py
@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+"""Test LZ4 interoperability between versions"""
+
+#
+# Copyright (C) 2011-present, Takayuki Matsuoka
+# All rights reserved.
+# GPL v2 License
+#
+
+import glob
+import subprocess
+import filecmp
+import os
+import shutil
+import sys
+import hashlib
+
+repo_url = 'https://github.com/lz4/lz4.git'
+tmp_dir_name = 'tests/abiTests'
+env_flags = ' ' # '-j MOREFLAGS="-g -O0 -fsanitize=address"'
+make_cmd = 'make'
+git_cmd = 'git'
+test_dat_src = ['README.md']
+head = 'v999'
+
+def proc(cmd_args, pipe=True, env=False):
+ if env == False:
+ env = os.environ.copy()
+ if pipe:
+ s = subprocess.Popen(cmd_args,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ env = env)
+ else:
+ s = subprocess.Popen(cmd_args, env = env)
+ r = s.communicate()
+ if s.poll() != 0:
+ print(' s.poll() = ', s.poll())
+ sys.exit(1)
+ return r
+
+def make(args, pipe=True, env=False):
+ if env == False:
+ env = os.environ.copy()
+ # we want the address sanitizer for abi tests
+ env["MOREFLAGS"] = "-fsanitize=address"
+ return proc([make_cmd] + ['-j'] + ['V=1'] + args, pipe, env)
+
+def git(args, pipe=True):
+ return proc([git_cmd] + args, pipe)
+
+def get_git_tags():
+ # Only start from first v1.7.x format release
+ stdout, stderr = git(['tag', '-l', 'v[1-9].[0-9].[0-9]'])
+ tags = stdout.decode('utf-8').split()
+ return tags
+
+# https://stackoverflow.com/a/19711609/2132223
+def sha1_of_file(filepath):
+ with open(filepath, 'rb') as f:
+ return hashlib.sha1(f.read()).hexdigest()
+
+if __name__ == '__main__':
+ error_code = 0
+ base_dir = os.getcwd() + '/..' # /path/to/lz4
+ tmp_dir = base_dir + '/' + tmp_dir_name # /path/to/lz4/tests/versionsTest
+ clone_dir = tmp_dir + '/' + 'lz4' # /path/to/lz4/tests/versionsTest/lz4
+ lib_dir = base_dir + '/lib' # /path/to/lz4/lib
+ test_dir = base_dir + '/tests'
+ os.makedirs(tmp_dir, exist_ok=True)
+
+ # since Travis clones limited depth, we should clone full repository
+ if not os.path.isdir(clone_dir):
+ git(['clone', repo_url, clone_dir])
+
+ # Retrieve all release tags
+ print('Retrieve release tags >= v1.7.5 :')
+ os.chdir(clone_dir)
+ tags = [head] + get_git_tags()
+ tags = [x for x in tags if (x >= 'v1.7.5')]
+ print(tags)
+
+ # loop across architectures
+ for march in ['-m64', '-m32', '-mx32']:
+ print(' ')
+ print('=====================================')
+ print('Testing architecture ' + march);
+ print('=====================================')
+
+ # Build all versions of liblz4
+ # note : naming scheme only works on Linux
+ for tag in tags:
+ print('building library ', tag)
+ os.chdir(base_dir)
+ # if not os.path.isfile(dst_liblz4) or tag == head:
+ if tag != head:
+ r_dir = '{}/{}'.format(tmp_dir, tag) # /path/to/lz4/test/lz4test/<TAG>
+ #print('r_dir = ', r_dir) # for debug
+ os.makedirs(r_dir, exist_ok=True)
+ os.chdir(clone_dir)
+ git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'])
+ os.chdir(r_dir + '/lib') # /path/to/lz4/lz4test/<TAG>/lib
+ else:
+ # print('lib_dir = {}', lib_dir) # for debug
+ os.chdir(lib_dir)
+ make(['clean'])
+ build_env = os.environ.copy()
+ build_env["CFLAGS"] = march
+ build_env["MOREFLAGS"] = "-fsanitize=address"
+ make(['liblz4'], env=build_env)
+
+ print(' ')
+ print('******************************')
+ print('Round trip expecting current ABI but linking to older Dynamic Library version')
+ print('******************************')
+ os.chdir(test_dir)
+ # Start with matching version : should be no problem
+ build_env = os.environ.copy()
+ build_env["CFLAGS"] = march
+ build_env["LDFLAGS"] = "-L../lib"
+ build_env["LDLIBS"] = "-llz4"
+ # we use asan to detect any out-of-bound read or write
+ build_env["MOREFLAGS"] = "-fsanitize=address"
+ if os.path.isfile('abiTest'):
+ os.remove('abiTest')
+ make(['abiTest'], env=build_env, pipe=False)
+
+ for tag in tags:
+ print('linking to lib tag = ', tag)
+ run_env = os.environ.copy()
+ if tag == head:
+ run_env["LD_LIBRARY_PATH"] = '../lib'
+ else:
+ run_env["LD_LIBRARY_PATH"] = 'abiTests/{}/lib'.format(tag)
+ # check we are linking to the right library version at run time
+ proc(['./check_liblz4_version.sh'] + ['./abiTest'], pipe=False, env=run_env)
+ # now run with mismatched library version
+ proc(['./abiTest'] + test_dat_src, pipe=False, env=run_env)
+
+ print(' ')
+ print('******************************')
+ print('Round trip using current Dynamic Library expecting older ABI version')
+ print('******************************')
+
+ for tag in tags:
+ print(' ')
+ print('building using older lib ', tag)
+ build_env = os.environ.copy()
+ if tag != head:
+ build_env["CPPFLAGS"] = '-IabiTests/{}/lib'.format(tag)
+ build_env["LDFLAGS"] = '-LabiTests/{}/lib'.format(tag)
+ else:
+ build_env["CPPFLAGS"] = '-I../lib'
+ build_env["LDFLAGS"] = '-L../lib'
+ build_env["LDLIBS"] = "-llz4"
+ build_env["CFLAGS"] = march
+ build_env["MOREFLAGS"] = "-fsanitize=address"
+ os.remove('abiTest')
+ make(['abiTest'], pipe=False, env=build_env)
+
+ print('run with CURRENT library version (head)')
+ run_env = os.environ.copy()
+ run_env["LD_LIBRARY_PATH"] = '../lib'
+ # check we are linking to the right library version at run time
+ proc(['./check_liblz4_version.sh'] + ['./abiTest'], pipe=False, env=run_env)
+ # now run with mismatched library version
+ proc(['./abiTest'] + test_dat_src, pipe=False, env=run_env)
+
+
+ if error_code != 0:
+ print('ERROR')
+
+ sys.exit(error_code)
diff --git a/tests/test-lz4-list.py b/tests/test-lz4-list.py
index ce89757..fe11682 100644
--- a/tests/test-lz4-list.py
+++ b/tests/test-lz4-list.py
@@ -1,16 +1,17 @@
-#! /usr/bin/env python3
+#!/usr/bin/env python3
import subprocess
import time
import glob
import os
import tempfile
import unittest
+import sys
SIZES = [3, 11] # Always 2 sizes
MIB = 1048576
-LZ4 = os.path.dirname(os.path.realpath(__file__)) + "/../lz4"
+LZ4 = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../lz4")
if not os.path.exists(LZ4):
- LZ4 = os.path.dirname(os.path.realpath(__file__)) + "/../programs/lz4"
+ LZ4 = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../programs/lz4")
TEMP = tempfile.gettempdir()
@@ -19,26 +20,29 @@ class NVerboseFileInfo(object):
self.line = line_in
splitlines = line_in.split()
if len(splitlines) != 7:
- errout("Unexpected line: {}".format(line_in))
+ errout(f"Unexpected line: {line_in}")
self.frames, self.type, self.block, self.compressed, self.uncompressed, self.ratio, self.filename = splitlines
self.exp_unc_size = 0
# Get real file sizes
if "concat-all" in self.filename or "2f--content-size" in self.filename:
for i in SIZES:
- self.exp_unc_size += os.path.getsize("{}/test_list_{}M".format(TEMP, i))
+ self.exp_unc_size += os.path.getsize(f"{TEMP}/test_list_{i}M")
else:
uncompressed_filename = self.filename.split("-")[0]
- self.exp_unc_size += os.path.getsize("{}/{}".format(TEMP, uncompressed_filename))
- self.exp_comp_size = os.path.getsize("{}/{}".format(TEMP, self.filename))
+ self.exp_unc_size += os.path.getsize(f"{TEMP}/{uncompressed_filename}")
+ self.exp_comp_size = os.path.getsize(f"{TEMP}/{self.filename}")
class TestNonVerbose(unittest.TestCase):
@classmethod
def setUpClass(self):
self.nvinfo_list = []
- for i, line in enumerate(execute("{} --list -m {}/test_list_*.lz4".format(LZ4, TEMP), print_output=True)):
- if i > 0:
- self.nvinfo_list.append(NVerboseFileInfo(line))
+ test_list_files = glob.glob(f"{TEMP}/test_list_*.lz4")
+ # One of the files has 2 frames so duplicate it in this list to map each frame 1 to a single file
+ for i, filename in enumerate(test_list_files):
+ for i, line in enumerate(execute(f"{LZ4} --list -m {filename}", print_output=True)):
+ if i > 0:
+ self.nvinfo_list.append(NVerboseFileInfo(line))
def test_frames(self):
all_concat_frames = 0
@@ -80,7 +84,7 @@ class TestNonVerbose(unittest.TestCase):
def test_ratio(self):
for nvinfo in self.nvinfo_list:
if "--content-size" in nvinfo.filename:
- self.assertEqual(nvinfo.ratio, "{:.2f}%".format(float(nvinfo.exp_comp_size) / float(nvinfo.exp_unc_size) * 100), nvinfo.line)
+ self.assertEqual(nvinfo.ratio, f"{float(nvinfo.exp_comp_size) / float(nvinfo.exp_unc_size) * 100:.2f}%", nvinfo.line)
def test_uncompressed_size(self):
for nvinfo in self.nvinfo_list:
@@ -112,17 +116,19 @@ class TestVerbose(unittest.TestCase):
# we're only really interested in testing the output of the concat-all file.
self.vinfo_list = []
start = end = 0
- output = execute("{} --list -m -v {}/test_list_concat-all.lz4 {}/test_list_*M-lz4f-2f--content-size.lz4".format(LZ4, TEMP, TEMP), print_output=True)
- for i, line in enumerate(output):
- if line.startswith("test_list"):
- if start != 0 and end != 0:
- self.vinfo_list.append(VerboseFileInfo(output[start:end]))
- start = i
- if not line:
- end = i
+ test_list_SM_lz4f = glob.glob(f"{TEMP}/test_list_*M-lz4f-2f--content-size.lz4")
+ for i, filename in enumerate(test_list_SM_lz4f):
+ output = execute(f"{LZ4} --list -m -v {TEMP}/test_list_concat-all.lz4 {filename}", print_output=True)
+ for i, line in enumerate(output):
+ if line.startswith("test_list"):
+ if start != 0 and end != 0:
+ self.vinfo_list.append(VerboseFileInfo(output[start:end]))
+ start = i
+ if not line:
+ end = i
self.vinfo_list.append(VerboseFileInfo(output[start:end]))
# Populate file_frame_map as a reference of the expected info
- concat_file_list = glob.glob("/tmp/test_list_[!concat]*.lz4")
+ concat_file_list = glob.glob(f"{TEMP}/test_list_[!concat]*.lz4")
# One of the files has 2 frames so duplicate it in this list to map each frame 1 to a single file
for i, filename in enumerate(concat_file_list):
if "2f--content-size" in filename:
@@ -130,11 +136,11 @@ class TestVerbose(unittest.TestCase):
break
self.cvinfo = self.vinfo_list[0]
self.cvinfo.file_frame_map = concat_file_list
- self.cvinfo.compressed_size = os.path.getsize("{}/test_list_concat-all.lz4".format(TEMP))
+ self.cvinfo.compressed_size = os.path.getsize(f"{TEMP}/test_list_concat-all.lz4")
def test_filename(self):
for i, vinfo in enumerate(self.vinfo_list):
- self.assertRegex(vinfo.filename, "^test_list_.*({}/{})".format(i + 1, len(self.vinfo_list)))
+ self.assertRegex(vinfo.filename, f"^test_list_.*({i + 1}/{len(self.vinfo_list)})".format(i + 1, len(self.vinfo_list)))
def test_frame_number(self):
for vinfo in self.vinfo_list:
@@ -169,7 +175,7 @@ class TestVerbose(unittest.TestCase):
expected_size = os.path.getsize(self.cvinfo.file_frame_map[i])
self.assertEqual(self.cvinfo.frame_list[i]["compressed"], str(expected_size), self.cvinfo.frame_list[i]["line"])
total += int(self.cvinfo.frame_list[i]["compressed"])
- self.assertEqual(total, self.cvinfo.compressed_size, "Expected total sum ({}) to match {} filesize".format(total, self.cvinfo.filename))
+ self.assertEqual(total, self.cvinfo.compressed_size, f"Expected total sum ({total}) to match {self.cvinfo.filename} filesize")
def test_uncompressed(self):
for i, frame_info in enumerate(self.cvinfo.frame_list):
@@ -182,7 +188,7 @@ class TestVerbose(unittest.TestCase):
for i, frame_info in enumerate(self.cvinfo.frame_list):
if "--content-size" in self.cvinfo.file_frame_map[i]:
self.assertEqual(self.cvinfo.frame_list[i]['ratio'],
- "{:.2f}%".format(float(self.cvinfo.frame_list[i]['compressed']) / float(self.cvinfo.frame_list[i]['uncompressed']) * 100),
+ f"{float(self.cvinfo.frame_list[i]['compressed']) / float(self.cvinfo.frame_list[i]['uncompressed']) * 100:.2f}%",
self.cvinfo.frame_list[i]["line"])
@@ -191,7 +197,7 @@ def to_human(size):
if size < 1024.0:
break
size /= 1024.0
- return "{:.2f}{}".format(size, unit)
+ return f"{size:.2f}{unit}"
def log(text):
@@ -203,12 +209,12 @@ def errout(text, err=1):
exit(err)
-def execute(command, print_command=True, print_output=False, print_error=True, param_shell=True):
+def execute(command, print_command=True, print_output=False, print_error=True):
if os.environ.get('QEMU_SYS'):
- command = "{} {}".format(os.environ['QEMU_SYS'], command)
+ command = f"{os.environ['QEMU_SYS']} {command}"
if print_command:
log("> " + command)
- popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=param_shell)
+ popen = subprocess.Popen(command.split(" "), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout_lines, stderr_lines = popen.communicate()
stderr_lines = stderr_lines.decode("utf-8")
stdout_lines = stdout_lines.decode("utf-8")
@@ -220,14 +226,14 @@ def execute(command, print_command=True, print_output=False, print_error=True, p
if popen.returncode is not None and popen.returncode != 0:
if stderr_lines and not print_output and print_error:
print(stderr_lines)
- errout("Failed to run: {}\n".format(command, stdout_lines + stderr_lines))
+ errout(f"Failed to run: {command}, {stdout_lines + stderr_lines}\n")
return (stdout_lines + stderr_lines).splitlines()
def cleanup(silent=False):
- for f in glob.glob("{}/test_list*".format(TEMP)):
+ for f in glob.glob(f"{TEMP}/test_list*"):
if not silent:
- log("Deleting {}".format(f))
+ log(f"Deleting {f}")
os.unlink(f)
@@ -243,33 +249,33 @@ def generate_files():
# file format ~ test_list<frametype>-<no_frames>f<create-args>.lz4 ~
# Generate LZ4Frames
for i in SIZES:
- filename = "{}/test_list_{}M".format(TEMP, i)
- log("Generating {}".format(filename))
+ filename = f"{TEMP}/test_list_{i}M"
+ log(f"Generating {filename}")
datagen(filename, i * MIB)
for j in ["--content-size", "-BI", "-BD", "-BX", "--no-frame-crc"]:
- lz4file = "{}-lz4f-1f{}.lz4".format(filename, j)
- execute("{} {} {} {}".format(LZ4, j, filename, lz4file))
+ lz4file = f"{filename}-lz4f-1f{j}.lz4"
+ execute(f"{LZ4} {j} {filename} {lz4file}")
# Generate skippable frames
- lz4file = "{}-skip-1f.lz4".format(filename)
+ lz4file = f"{filename}-skip-1f.lz4"
skipsize = i * 1024
skipbytes = bytes([80, 42, 77, 24]) + skipsize.to_bytes(4, byteorder='little', signed=False)
with open(lz4file, 'wb') as f:
f.write(skipbytes)
f.write(os.urandom(skipsize))
# Generate legacy frames
- lz4file = "{}-legc-1f.lz4".format(filename)
- execute("{} -l {} {}".format(LZ4, filename, lz4file))
+ lz4file = f"{filename}-legc-1f.lz4"
+ execute(f"{LZ4} -l {filename} {lz4file}")
# Concatenate --content-size files
- file_list = glob.glob("{}/test_list_*-lz4f-1f--content-size.lz4".format(TEMP))
- with open("{}/test_list_{}M-lz4f-2f--content-size.lz4".format(TEMP, sum(SIZES)), 'ab') as outfile:
+ file_list = glob.glob(f"{TEMP}/test_list_*-lz4f-1f--content-size.lz4")
+ with open(f"{TEMP}/test_list_{sum(SIZES)}M-lz4f-2f--content-size.lz4", 'ab') as outfile:
for fname in file_list:
with open(fname, 'rb') as infile:
outfile.write(infile.read())
# Concatenate all files
- file_list = glob.glob("{}/test_list_*.lz4".format(TEMP))
- with open("{}/test_list_concat-all.lz4".format(TEMP), 'ab') as outfile:
+ file_list = glob.glob(f"{TEMP}/test_list_*.lz4")
+ with open(f"{TEMP}/test_list_concat-all.lz4", 'ab') as outfile:
for fname in file_list:
with open(fname, 'rb') as infile:
outfile.write(infile.read())
@@ -278,5 +284,6 @@ def generate_files():
if __name__ == '__main__':
cleanup()
generate_files()
- unittest.main(verbosity=2, exit=False)
+ ret = unittest.main(verbosity=2, exit=False)
cleanup(silent=True)
+ sys.exit(not ret.result.wasSuccessful())
diff --git a/tests/test-lz4-speed.py b/tests/test-lz4-speed.py
index ca8f010..658939c 100644
--- a/tests/test-lz4-speed.py
+++ b/tests/test-lz4-speed.py
@@ -1,7 +1,7 @@
-#! /usr/bin/env python3
+#!/usr/bin/env python3
#
-# Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+# Copyright (c) 2016-2020, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
@@ -152,7 +152,7 @@ def benchmark_and_compare(branch, commit, last_commit, args, executableName, md5
% (os.getloadavg()[0], args.maxLoadAvg, sleepTime))
time.sleep(sleepTime)
start_load = str(os.getloadavg())
- result = execute('programs/%s -rqi5b1e%s %s' % (executableName, args.lastCLevel, testFilePath), print_output=True)
+ result = execute('programs/%s -rqi5b1e%s %s' % (executableName, args.lastCLevel, testFilePath), print_output=True)
end_load = str(os.getloadavg())
linesExpected = args.lastCLevel + 1
if len(result) != linesExpected:
diff --git a/tests/test_custom_block_sizes.sh b/tests/test_custom_block_sizes.sh
index aba6733..2f6591f 100755
--- a/tests/test_custom_block_sizes.sh
+++ b/tests/test_custom_block_sizes.sh
@@ -1,4 +1,4 @@
-#/usr/bin/env sh
+#!/usr/bin/env sh
set -e
LZ4=../lz4
@@ -62,7 +62,7 @@ do
done
rm $TMPFILE.lz4 $TMPFILE1 $TMPFILE1.lz4 $TMPFILE2 $TMPFILE2.lz4
-if [ "$failures" == "" ]
+if [ "$failures" = "" ]
then
echo ---- All tests passed
exit 0
diff --git a/tests/test_install.sh b/tests/test_install.sh
index 122bac5..ad0d4eb 100755
--- a/tests/test_install.sh
+++ b/tests/test_install.sh
@@ -1,4 +1,4 @@
-#/usr/bin/env sh
+#!/usr/bin/env sh
set -e
@@ -6,7 +6,7 @@ make="make -C $lz4_root"
unamestr=$(uname)
if [ "$unamestr" = 'Linux' ]; then
make="make -C $lz4_root"
-elif [ "$unamestr" = 'FreeBSD' -o "$unamestr" = 'OpenBSD' ]; then
+elif [ "$unamestr" = 'FreeBSD' ] || [ "$unamestr" = 'OpenBSD' ]; then
make="gmake -C $lz4_root"
fi
diff --git a/tests/unicode_lint.sh b/tests/unicode_lint.sh
new file mode 100644
index 0000000..cd65d2d
--- /dev/null
+++ b/tests/unicode_lint.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# `unicode_lint.sh' determines whether source files under the ./lib/, ./tests/ and ./programs/ directories
+# contain Unicode characters, and fails if any do.
+#
+# See https://github.com/lz4/lz4/issues/1018
+
+echo "Ensure no unicode character is present in source files *.{c,h}"
+pass=true
+
+# Scan ./lib/ for Unicode in source (*.c, *.h) files
+echo "Scanning lib/"
+result=$(
+ find ./lib/ -regex '.*\.\(c\|h\)$' -exec grep -P -n "[^\x00-\x7F]" {} \; -exec echo "{}: FAIL" \;
+)
+if [[ $result ]]; then
+ echo "$result"
+ pass=false
+fi
+
+# Scan ./programs/ for Unicode in source (*.c, *.h) files
+echo "Scanning programs/"
+result=$(
+ find ./programs/ -regex '.*\.\(c\|h\)$' -exec grep -P -n "[^\x00-\x7F]" {} \; -exec echo "{}: FAIL" \;
+)
+if [[ $result ]]; then
+ echo "$result"
+ pass=false
+fi
+
+# Scan ./tests/ for Unicode in source (*.c, *.h) files
+echo "Scanning tests/"
+result=$(
+ find ./tests/ -regex '.*\.\(c\|h\)$' -exec grep -P -n "[^\x00-\x7F]" {} \; -exec echo "{}: FAIL" \;
+)
+if [[ $result ]]; then
+ echo "$result"
+ pass=false
+fi
+
+if [ "$pass" = true ]; then
+ echo "All tests successful: no unicode character detected"
+ echo "Result: PASS"
+ exit 0
+else
+ echo "Result: FAIL"
+ exit 1
+fi
generated by cgit v0.12 at 2024-06-25 23:02:51 (GMT)